def image_download(self): queue_endian = self.queue_endian threads_num = self.num_threads download_item_queue = ProcQueue() save_path = self.save_path download_item_dict = self.download_item_dict logger.debug("Item Num Producer") for goodscode in download_item_dict: item_id = str(goodscode) input_data = [save_path, item_id] download_item_queue.put(input_data) for i in range(0, threads_num): download_item_queue.put(queue_endian) run_threads = list() logger.debug("Item Image URL check") for i in range(0, threads_num): __single_thread = ImageDownloaderWorker(download_item_queue, queue_endian) run_threads.append(__single_thread) __single_thread.start() for __single_thread in run_threads: __single_thread.join() logger.debug("Close Queues") download_item_queue.close() download_item_queue.join_thread()
class ProcessExecutor(object): def __init__(self): self.processes = [] self.q = Queue() def wait_until_finished(self): while self.processes: processes = self.processes self.processes = [] [_process.join() for _process in processes] self.q.close() self.q.join_thread() def clean(self): self.processes = [] def execute(self, fn, *args, **kwargs): promise = Promise() self.q.put([promise, fn, args, kwargs], False) _process = Process(target=queue_process, args=(self.q)) _process.start() self.processes.append(_process) return promise
def test_tiny_circuit(): t = 2 n = 5 c_path = "circuits/tiny_circuit.txt" triples = gen_triples(t, n, 2) mq = Queue() queues = [Queue() for _ in range(n)] x = '01' y = '10' result = '11' x_shares = Shamir(t, n).share_bitstring_secret(x) y_shares = Shamir(t, n).share_bitstring_secret(y) processes = [] for i in range(n): p = Process(target=run_circuit_process, args=(t, n, c_path, i + 1, queues, mq, x_shares[i] + y_shares[i], triples[i])) processes.append(p) for p in processes: p.start() vals = [] while len(vals) < n: if not mq.empty(): vals.append(mq.get()) reconstructed = Shamir(t, n).reconstruct_bitstring_secret(vals) print("result:", reconstructed) assert reconstructed == result, "result incorrect" print("TEST PASSED") for p in processes: p.join() for q in queues: q.close() q.join_thread() mq.close() mq.join_thread()
def main(source): """ main method""" print("Starting parse of file %s at %s" %(source, time.ctime())) loadAuthorFilter() start_time = time.time() queue = Queue() process_list = list() for _ in range(4): process_list.append(Process(target=insertIntoGraph, args=(queue,))) #process_list.append(Process(target=createCypherFiles, args=(queue,))) for process in process_list: process.start() source = gzip.open(source, mode='rt', encoding='latin-1') parser = make_parser() parser.setContentHandler(DBLPContentHandler(queue)) parser.parse(source) queue.join_thread() #TODO: Make a test, with this instructions all processes never end. #for process in process_list: # process.join() # logging.info("Consumer terminated") elapsed_time = time.time() - start_time print("Execution time ", datetime.timedelta(seconds=elapsed_time)) print("Finish")
def main(): # create thread for each request queue = Queue() datas = [] res = [] threads = [] for i in range(numReqs): if (i == numReqs - 1): th = threading.Thread(target=sendRequest, args=(None, queue)) elif ((i + 1) % 2 == 0): data = json.dumps({'action': 'jump', 'time': i + 1}) datas.append(data) th = threading.Thread(target=sendRequest, args=(data, queue)) elif ((i + 1) % 5 == 0): data = json.dumps({'action': 'swim', 'time': i + 1}) datas.append(data) th = threading.Thread(target=sendRequest, args=(data, queue)) else: th = threading.Thread(target=sendRequest, args=(None, queue)) th.start() res.append(queue.get()) # need queue delay when numReqs is small if (numReqs < 10): time.sleep(0.1) queue.close() queue.join_thread() for t in threads: t.join() # evaluate res if (evalAvg(datas, res[len(res) - 1])): print 'Success' else: print 'Failed'
def test_mpc(t, n, c_path, n_triples, inputs, result, reflect=False): triples = gen_triples(t, n, n_triples) mq = Queue() queues = [Queue() for _ in range(n)] share_inputs = [Shamir(t, n).share_bitstring_secret(i) for i in inputs] processes = [] for i in range(n): inputs = [] for si in share_inputs: inputs.extend(si[i]) p = Process(target=run_circuit_process, args=(t, n, c_path, i+1, queues, mq, inputs, triples[i])) processes.append(p) start = time.time() for p in processes: p.start() t1 = Thread(target=consumer, args=(mq, n, result, t, processes, reflect)) t1.start() for p in processes: if p.is_alive(): p.join() t1.join(n) print(f"time: {round(time.time()-start, 4)} seconds") for q in queues: q.close() q.join_thread() mq.join_thread()
def email_campaign_manager_process( *, campaign: Campaign, queue: mp.Queue, quit_event: mp.Event ): """ Main function of the process responsible for scheduling the sending of campaigns :param campaign: the campaign for which messages must be sent :type campaign: :class:`nuntius.models.Campaign` :param queue: the work queue on which email messages are put :type queue: : class:`multiprocessing.Queue` :param quit_event: an event that may be used by the main process to tell the manager it needs to quit :type quit_event: class:`multiprocessing.Event` """ queryset = campaign.get_subscribers_queryset() # eliminate people who already received the message queryset = queryset.annotate( already_sent=Exists( CampaignSentEvent.objects.filter( subscriber_id=OuterRef("pk"), campaign_id=campaign.id ).exclude(result=CampaignSentStatusType.PENDING) ) ).filter(already_sent=False) campaign_finished = False for subscriber in queryset.iterator(): if quit_event.is_set(): break if subscriber.get_subscriber_status() != AbstractSubscriber.STATUS_SUBSCRIBED: continue sent_event = campaign.get_event_for_subscriber(subscriber) # just in case there is another nuntius_worker started, but this should not happen if sent_event.result != CampaignSentStatusType.PENDING: continue message = message_for_event(sent_event) try: put_in_queue_or_quit( queue, (message, sent_event.id), event=quit_event, polling_period=app_settings.POLLING_INTERVAL, ) except GracefulExit: break else: campaign_finished = True queue.close() queue.join_thread() # everything has been scheduled for sending: if campaign_finished: campaign.status = Campaign.STATUS_SENT campaign.save()
def find_img_pos_multi(screen, img): s = time.time() H, W = screen.shape[0:2] h, w = img.shape[0:2] min_diff = 10000 pos = np.array([0, 0]) all_pixel_num = (H - h + 1) * (W - w + 1) Ws = [0, 1 * W // 4, 2 * W // 4, 3 * W // 4, W] result = Queue() procs = [] for i in range(4): proc = Process(target=TargetFunction.find_img_pos_multi_target, args=(screen, img, result, Ws[i], Ws[i + 1])) procs.append(proc) proc.start() results = [] for i in range(4): results.append(result.get()) pos, min_diff = min(results, key=lambda x: x[1]) result.close() result.join_thread() for proc in procs: proc.join() # print(time.time() - s) return pos, min_diff
def sentence_checker(self, sentence, context_sentences, q): print 'Process webExtracter.sentence_checker running...' jobs = [] scrapped_data = {} query = sentence for j in search(query, tld="co.in", num=5, stop=1, pause=2): q = Queue() z = webExtracter() p = Process(target=z.scrapper, args=(j, q)) jobs.append(p) trusted_sources = [r'quora'] for source in trusted_sources: if re.search(source, j): start = time.clock() p.start() p.join() print 'time taken for this process is ', time.clock( ) - start scrapped_data[j] = q.get() break else: p.start() p.join(2) scrapped_data[j] = q.get() #print 7 if p.is_alive(): print "this shit is still running... let's kill it..." p.terminate() p.join() scrapped_data[j] = 'KILLED' p = webExtracter() match_q = p.Data_Processing(scrapped_data, query, context_sentences) q.put(match_q) q.close() q.join_thread()
class _workerQpushTimer(): def __init__(self): self.syncPeriod = 2 self.timer = None self.Qinit() def Qinit(self): self.syncTmpQ = Queue() # flush remain items in queue, and then close and join_thread def Qflush(self): while True: try: self.syncTmpQ.get(True, comm.FLUSH_TIMEOUT) except Empty: break self.syncTmpQ.close() self.syncTmpQ.join_thread() def enableTimer(self, workerPool): self.timer = Timer(self.syncPeriod, self.pushToWorkerQ, [workerPool]) self.timer.start() def disableTimer(self): if self.timer is not None: self.timer.cancel() # function executed periodically, used to sync queue between main process queue and worker queue def pushToWorkerQ(self, workerPool): while not comm.done.value: try: item = self.syncTmpQ.get_nowait() for w in workerPool: w.queue.put_nowait(item) except Empty: break if not comm.done.value: self.enableTimer(workerPool)
class Eater: """ Worker which does work in another process so that the current process can continue. Calls to apply can not return anything and return immediately. """ def __init__(self, worker, n_jobs=-1, max_size=-1): self.worker = worker if n_jobs == -1: n_jobs = cpu_count() self.n_jobs = n_jobs self.max_size = max_size self.writeto = Queue(self.max_size) self.processes = [ Process(target=TargetWrapper(self.worker, self.writeto, returns=False), name='EaterPool {0}'.format(i)) for i in range(self.n_jobs) ] for process in self.processes: process.start() def eat(self, item): self.writeto.put((True, item)) def close(self): for _ in self.processes: self.writeto.put((False, )) self.writeto.close() self.writeto.join_thread()
def test_mpc(t, n, c_path, n_triples, all_inputs, result, reflect=False): triples = gen_triples(t, n, n_triples) mq = Queue() queues = [Queue() for _ in range(n)] processes = [] itypes = [] for i in all_inputs[0]: if i in [0, 1]: itypes.append('V') else: itypes.append('S') c = Circuit(c_path, itypes) for i in range(n): p = Process(target=run_circuit_process, args=(t, n, c, i + 1, queues, mq, all_inputs[i], triples[i])) processes.append(p) start = time.time() for p in processes: p.start() t1 = Process(target=consumer, args=(mq, n, result, t, processes, reflect)) t1.start() for p in processes: p.join() t1.join() print(f"time: {round(time.time()-start, 4)} seconds") while not mq.empty(): mq.get() mq.close() for q in queues: q.close() q.join_thread() mq.join_thread()
def main(): my_list = list(range(20000000)) q = Queue() p1 = Process(target=do_sum, args=(q, my_list[:5000000])) p2 = Process(target=do_sum, args=(q, my_list[5000000:10000000])) p3 = Process(target=do_sum, args=(q, my_list[10000000:15000000])) p4 = Process(target=do_sum, args=(q, my_list[15000000:])) p1.start() p2.start() p3.start() p4.start() r1 = q.get() r2 = q.get() r3 = q.get() r4 = q.get() q.close() q.join_thread() p1.join() p2.join() p3.join() p4.join() print(r1 + r2 + r3 + r4) # pylint: disable=print-statement
def main(connection_string): overall_start_time = time.time() session = setup_connection(connection_string, create_db=True) for entry in FILELIST: global CURRENT_FILENAME CURRENT_FILENAME = entry f_name = f"./databases/{entry}" if os.path.exists(f_name): logger.info(f"parsing database file: {f_name}") start_time = time.time() blocks = read_blocks(f_name) logger.info( f"database parsing finished: {round(time.time() - start_time, 2)} seconds" ) logger.info('parsing blocks') start_time = time.time() jobs = Queue() workers = [] # start workers logger.debug(f"starting {NUM_WORKERS} processes") for w in range(NUM_WORKERS): p = Process(target=parse_blocks, args=( jobs, connection_string, ), daemon=True) p.start() workers.append(p) # add tasks for b in blocks: jobs.put(b) for i in range(NUM_WORKERS): jobs.put(None) jobs.close() jobs.join_thread() # wait to finish for p in workers: p.join() logger.info( f"block parsing finished: {round(time.time() - start_time, 2)} seconds" ) else: logger.info( f"File {f_name} not found. Please download using download_dumps.sh" ) CURRENT_FILENAME = "empty" logger.info( f"script finished: {round(time.time() - overall_start_time, 2)} seconds" )
def start(self): # print(tf.config.experimental.list_physical_devices(device_type=None)) # print(tf.config.experimental.list_logical_devices(device_type=None)) self.episode_num = 200 self.ps_num = 1 self.worker_num = 3 self.current_episode = 1 global_remain_episode = Value('i', self.episode_num) global_alive_workers = Value('i', self.worker_num) global_res_queue = Queue() global_grad_queue = Queue() global_var_queues = [Queue(1) for i in range(self.worker_num)] cluster_config = self.make_cluster_config(self.ps_num, self.worker_num) pss = [] workers = [] episode_results = [] cluster_spec = tf.train.ClusterSpec(cluster_config) for ps_id in range(self.ps_num): pss.append(Process(target = self.param_server, args=(ps_id, ps_id, global_remain_episode, global_alive_workers, global_grad_queue, global_var_queues))) for worker_id in range(self.worker_num): workers.append(Process(target = self.worker, args=(worker_id + self.ps_num, worker_id, global_remain_episode, global_alive_workers, global_grad_queue, global_var_queues[worker_id], global_res_queue))) for num in range(self.ps_num): pss[num].start() for num in range(self.worker_num): workers[num].start() while ((not global_res_queue.empty()) or (global_alive_workers.value > 0)): if not global_res_queue.empty(): episode_results.append(global_res_queue.get()) episode_res = episode_results.pop(0) print(f"Episode {self.current_episode} Reward with worker {episode_res['worker_id']}: {episode_res['reward']}\t| Loss: {episode_res['loss']}") self.current_episode += 1 global_grad_queue.close() global_grad_queue.join_thread() global_res_queue.close() global_res_queue.join_thread() for queue in global_var_queues: queue.close() queue.join_thread() for num in range(self.worker_num): workers[num].join() print(f'Worker {num} join') for num in range(self.ps_num): pss[num].join() print(f'PS {num} join')
def run_workers(self): '''Runs workers by assigning them a product index relative to the total number of products in the search query (starting from 0). Checks for any resume point and continues from there. Each Worker is given a product index to download, and is executed in a seperate Process. Free workers are queued in process.Queue ''' # create locks request_lock = Lock() log_download_progress_lock = Lock() # loading resume points self.query = query_formatter.adjust_for_specific_product(self.query) ready_worker_queue = Queue(maxsize=len(self.worker_list)) resume_point = self.load_resume_point() if resume_point == None: resume_point = 0 else: resume_point = int(resume_point) for worker in self.worker_list: worker.register_settings(self.query, self.download_location, self.polling_interval, self.offline_retries) try: worker_resume_point = worker.load_resume_point() if worker_resume_point == None: ready_worker_queue.put_nowait(worker) else: p = Process(target=worker.run_in_seperate_process, args=(worker_resume_point, ready_worker_queue, request_lock, log_download_progress_lock)) p.start() except Exception as e: self.logger.error(e) self.logger.error(Fore.RED + "Error in queueing workers") # assigning workers a result number to download for i in range(resume_point, int(self.total_results)): worker = ready_worker_queue.get() if worker.return_msg != None: self.logger.info(worker.return_msg) p = Process(target=worker.run_in_seperate_process, args=(i, ready_worker_queue, request_lock, log_download_progress_lock)) p.start() resume_point += 1 self.update_resume_point(resume_point) # clearing the last batch for i in range(0, len(self.worker_list)): worker = ready_worker_queue.get() if worker.return_msg: self.logger.info(worker.return_msg) ready_worker_queue.close() ready_worker_queue.join_thread() self._close_all_loggers() self.logger.info("Exiting...")
def processor(): master_list = [] master_dict = {} start = time.time() process_list = [] p_num = int(cpu_count() / 2) with Pool(processes=p_num) as pool: x = int(8 / p_num) definer = lambda test_list, x: [ test_list[i:i + x] for i in range(0, len(test_list), x) ] lists_per_core = definer(list(range(8)), x) pools = Queue() sub_master_list = Queue() pools.put(lists_per_core) sub_master_list.put({'placeholder': 0}) #IMPORTANT !!!!!!!!!!! for i in range(p_num): process_list.append( Process(target=assigner, args=( i, pools, sub_master_list, ))) process_list[i].start() pools.close() sub_master_list.close() pools.join_thread() sub_master_list.join_thread() for p in process_list: p.join() master_list = list(sub_master_list) print('Done') print(f'Time elapsed since launch {time.time()-start}') for i in range(8): master_dict = merger(master_dict, master_list[i], i) print('Done') master_dict = aver_maker(master_dict) print(master_dict) print(f'Time elapsed since launch {time.time()-start}')
def main(): q = Queue(10) p1 = Process(target=func_put, args=(q, )) p1.start() p2 = Process(target=func_get, args=(q, )) p2.start() p1.join() q.close() p2.join() q.join_thread()
def test_transaction_large(self) -> None: queue = Queue() # type: Queue[str] msg = 't' * 100001 # longer than the max read size of 100_000 p = Process(target=server, args=(msg, queue), daemon=True) p.start() connection_name = queue.get() with IPCClient(connection_name, timeout=1) as client: assert client.read() == msg.encode() client.write(b'test') queue.close() queue.join_thread() p.join()
class BaseWorker: name = 'base' def __init__(self): self._command_queue = Queue() self._result_queue = Queue() self._stop_requested = Event() self._command_queue_lock = Lock() self._process = Process(target=_run, args=(self._command_queue, self._result_queue, self._stop_requested)) def __enter__(self): self.run() return self def __exit__(self, *args): self.stop() def run(self): logger.info('starting %s worker', self.name) self._process.start() def stop(self): logger.info('stopping %s worker', self.name) # unblock the command_queue in the worker self._stop_requested.set() # close both queues self._command_queue.close() self._command_queue.join_thread() self._result_queue.close() self._result_queue.join_thread() # wait for the worker process to stop if self._process.is_alive(): self._process.join() logger.info('%s worker stopped', self.name) def send_cmd_and_wait(self, cmd, *args, **kwargs): if not self._process.is_alive(): logger.info('%s process is dead quitting', self.name) # kill the main thread os.kill(os.getpid(), signal.SIGTERM) # shutdown the current thread execution so that executor.shutdown does not block sys.exit(1) with self._command_queue_lock: self._command_queue.put((cmd, args, kwargs)) return self._result_queue.get()
def main(): # build_proxy() queue = Queue(2048) for i in range(128): Process(target=retrieve_from_queue, args=(queue, )).start() with open('samples.log') as f: process(f, queue) queue.close() queue.join_thread() # pool.close() pool.join()
class Layer2Worker(): def __init__(self): self.queue = Queue() # close and join_thread queue def Qflush(self): while True: time.sleep(comm.FLUSH_TIMEOUT) if self.queue.qsize() == 0: break self.queue.close() self.queue.join_thread() def saveProcess(self, process): self.process = process
def start(self, port, msgs: multiprocessing.Queue, inbound_msgs, key): self.msgs = msgs self.key = key #f = io.StringIO() #with redirect_stdout(f): #web.run_app(self.app, host='127.0.0.1', port=port, # handle_signals=True) loop = asyncio.get_event_loop() loop.run_until_complete(self._run_server(port, inbound_msgs)) msgs.close() inbound_msgs.close() msgs.join_thread() inbound_msgs.join_thread()
def main(): # build_proxy() queue = Queue(2048) for i in range(128): Process(target=retrieve_from_queue, args=(queue,)).start() with open('samples.log') as f: process(f, queue) queue.close() queue.join_thread() # pool.close() pool.join()
def get_item_url_check(self): logger.debug("URL Item Exist Check") item_dict = self.item_dict item_dict_key = item_dict.keys() queue_endian = self.queue_endian threads_num = self.num_threads good_item_queue = ProcQueue() good_item_result_queue = ProcQueue() logger.debug("Item Num Producer") for goodscode in item_dict_key: item_id = str(goodscode) good_item_queue.put(item_id) for i in range(0, threads_num): good_item_queue.put(queue_endian) run_threads = list() logger.debug("Item Image URL check") for i in range(0, threads_num): __single_thread = ImageCheckWorker(good_item_queue, good_item_result_queue, queue_endian) run_threads.append(__single_thread) __single_thread.start() for __single_thread in run_threads: __single_thread.join() # good_item_result_queue.put(queue_endian) logger.debug("Get Null Item List") null_image_list = ImageResultSum(good_item_result_queue, queue_endian, threads_num) logger.debug("Remove Null Item List") for null_item in null_image_list: del item_dict[null_item] logger.debug("Close Queues") good_item_queue.close() good_item_queue.join_thread() good_item_result_queue.close() good_item_result_queue.join_thread() self.item_dict = item_dict print(str(self.item_dict)) return self.item_dict
def test_queue(): q=Queue() #procLst=[Process(target=p, args=(q,) ) for p in [prod1, prod2, consum1, consum2]] pLst=[Process(target=p, args=(q,) ) for p in [prod1, prod2]] cLst=[Process(target=p, args=(q,) ) for p in [consum1, consum2]] procLst=pLst+cLst for pp in procLst: pp.start() # for pp in pLst: # pp.join() # q.put('STOP') q.close() # print 'Queue is closed' q.join_thread()
def video_capture(): """ Video capture. """ # Default video capture cap = cv2.VideoCapture(0) if not cap.isOpened(): logging.error("Cannot open camera") exit() input_q = Queue() output_q = Queue() process = Process(target=object_detection, args=(input_q, output_q)) process.start() processed_frame = None cv2.namedWindow("main") cv2.namedWindow("object") while True: # Capture frame-by-frame ret, frame = cap.read() # If frame is read correctly ret is True if not ret: logging.error("Can't receive frame. Exiting ...") break if input_q.empty(): input_q.put(frame) concat_frame = frame if not output_q.empty(): processed_frame = output_q.get() cv2.imshow("object", processed_frame) cv2.imshow("main", frame) if cv2.waitKey(1) == ord("q"): input_q.close() output_q.close() input_q.join_thread() output_q.join_thread() process.terminate() break # When everything done, release the capture cap.release() cv2.destroyAllWindows()
class Director(object): def __init__(self, producer, consumer): self.producer = producer self.consumer = consumer self.queue = Queue() self.prod_proc = Process(target = self.produce) self.prod_proc.daemon = True self.lock = Lock() self.done = Value('b') self.done.value = 0 def start(self): self.prod_proc.start() def step(self): self.lock.acquire() done = (self.done.value != 0) self.lock.release() if done: raise Done try: data = self.queue.get(block = True, timeout = 1.0) self.consumer.consume(data) except Empty: pass def stop(self): self.prod_proc.join() def run(self): self.start() while True: try: self.step() except Done: break self.stop() def produce(self): try: while True: data = self.producer.produce() self.queue.put(data) except: self.lock.acquire() self.done.value = 1 self.lock.release() self.queue.close() self.queue.join_thread()
class Director(object): def __init__(self, producer, consumer): self.producer = producer self.consumer = consumer self.queue = Queue() self.prod_proc = Process(target=self.produce) self.prod_proc.daemon = True self.lock = Lock() self.done = Value('b') self.done.value = 0 def start(self): self.prod_proc.start() def step(self): self.lock.acquire() done = (self.done.value != 0) self.lock.release() if done: raise Done try: data = self.queue.get(block=True, timeout=1.0) self.consumer.consume(data) except Empty: pass def stop(self): self.prod_proc.join() def run(self): self.start() while True: try: self.step() except Done: break self.stop() def produce(self): try: while True: data = self.producer.produce() self.queue.put(data) except: self.lock.acquire() self.done.value = 1 self.lock.release() self.queue.close() self.queue.join_thread()
def _worker(worker_class: Type[Worker], input_queue: Queue, output_queue: Queue, num_active_workers: Value, worker_id: int, kwargs=None) -> None: """ A worker that pulls data pints off the input queue, and places the execution result on the output queue. When there are no data pints left on the input queue, it decrements num_active_workers to signal completion. """ if kwargs is None: kwargs = {} logger.info(f"Reader worker: {worker_id} PID: {os.getpid()}") try: worker = worker_class.start(**kwargs) # Keep going until you get an item that's None. def input_queue_iterable() -> Iterable[Any]: while True: item = input_queue.get() if item == QueueSignals.stop: break yield item for processed_item in worker.process(input_queue_iterable()): output_queue.put(processed_item) except Exception as e: # pylint: disable=broad-except logger.exception(e) output_queue.put(QueueSignals.error) finally: # It's important that we close and join the queue here before # decrementing num_active_workers. Otherwise our parent may join us # before the queue's feeder thread has passed all buffered items to # the underlying pipe resulting in a deadlock. # # See: # https://docs.python.org/3.6/library/multiprocessing.html?highlight=process#pipes-and-queues # https://docs.python.org/3.6/library/multiprocessing.html?highlight=process#programming-guidelines output_queue.close() output_queue.join_thread() with num_active_workers.get_lock(): num_active_workers.value -= 1 logger.info(f"Reader worker {worker_id} finished")
def video_capture(): os.system(camSetupScript) frame_width = 640 frame_height = 480 stream = cv2.VideoCapture(0) stream.set(cv2.CAP_PROP_FRAME_WIDTH, CAPTURE_WIDTH) stream.set(cv2.CAP_PROP_FRAME_HEIGHT, CAPTURE_HEIGHT) start = time.time() timeElapsed = 0 frameCount = 0 def worker(q): out = cv2.VideoWriter(filename + ".avi", cv2.VideoWriter_fourcc(*"MJPG"), 30.0, (frame_width, frame_height)) while q.empty(): continue count = 0 while not q.empty(): out.write(q.get()) count += 1 if count % 100 == 0: print("Wrote ", count, " frames.") out.release() print("Finished writing.") queue = Queue() p = Process(target=worker, args=(queue, )) p.daemon = True p.start() while timeElapsed < 1000 / 30: # 1000 frames. ret, frame = stream.read() if ret: queue.put(frame) frameCount += 1 if frameCount % 100 == 0: print("Read ", frameCount, " frames.") timeElapsed = time.time() - start print('Total frames: ', frameCount) p.join() queue.close() queue.join_thread() print("Shutting down.") stream.release()
def start(test): if not test: print("Please provide a value for --test option") exit(1) if test == "with_update_lock": target = with_update_lock elif test == "with_optimistic_lock": target = with_optimistic_lock elif test == "with_serializable": target = with_serializable else: print("Bad argument value for test") exit(1) signal.signal(signal.SIGINT, handle_sigint) print(target.__name__) q = Queue() processes = [] no_of_processes = 2 for _ in range(no_of_processes): process = Process(target=target, args=(q, )) processes.append(process) process.start() q.put(False) time.sleep(1) while True: if kill: for _ in range(no_of_processes): q.put(True) break dead = True for p in processes: if p.is_alive(): dead = False if dead: break q.close() q.join_thread() for process in processes: process.join()
def test_run_flush_retire(self, fake_process_data, fake_get_logger, fake_flush_on_term): """``Worker`` 'run' flushes any data before retiring""" idle_queue = MagicMock() work_queue = Queue() work_queue.put(worker.SENTINEL) w = DerpWorker(work_group='testing', work_queue=work_queue, idle_queue=idle_queue) w.run() work_queue.close() work_queue.join_thread() called_fake_flush_on_term = fake_flush_on_term.called self.assertTrue(called_fake_flush_on_term)
def test_connect_twice(self) -> None: queue = Queue() # type: Queue[str] msg = 'this is a test message' p = Process(target=server, args=(msg, queue), daemon=True) p.start() connection_name = queue.get() with IPCClient(connection_name, timeout=1) as client: assert client.read() == msg.encode() client.write(b'') # don't let the server hang up yet, we want to connect again. with IPCClient(connection_name, timeout=1) as client: assert client.read() == msg.encode() client.write(b'test') queue.close() queue.join_thread() p.join() assert p.exitcode == 0
def test_connect_twice(self) -> None: queue = Queue() # type: Queue[str] msg = 'this is a test message' p = Process(target=server, args=(msg, queue), daemon=True) p.start() connection_name = queue.get() with IPCClient(connection_name, timeout=1) as client: assert client.read() == msg.encode() client.write( b'' ) # don't let the server hang up yet, we want to connect again. with IPCClient(connection_name, timeout=1) as client: client.write(b'test') queue.close() queue.join_thread() p.join()
def test_run_processes_data(self, fake_process_data, fake_get_logger): """``Worker`` the 'run' method calls 'process_data' with what is pulls form the work_queue""" idle_queue = MagicMock() work_queue = Queue() work_queue.put('some Work') work_queue.put(worker.SENTINEL) w = DerpWorker(work_group='testing', work_queue=work_queue, idle_queue=idle_queue) w.run() work_queue.close() work_queue.join_thread() called_process_data = fake_process_data.called self.assertTrue(called_process_data)
def main(): #build_proxy() #pool = Pool(processes=512) #m = Manager() queue = Queue(2048) pool = []*10 for i in range(128): p = Process(target=retrieve_from_queue, args=(queue,)) p.start() exist_file = 0 socket.setdefaulttimeout(3) with open('samples.log') as f: for index, line in enumerate(f): if index % 1000 == 0: print index, line try: args = line.split() if len(args) == 2: count, url = args else: count = args[0] url = ''.join(args[1:]) except Exception as e: print 'exception:', str(e), '|', line continue # print 'main:', count, url fname = urlparse(url).path.split('/')[-1] path = './imgs/'+str(index)+'.'+count+'.'+fname ''' result = pool.apply_async( retrieve, args=(url, path, queue), callback=callback ) ''' queue.put((url, path)) print 'apply async done' queue.close() queue.join_thread() # pool.close() pool.join() for e in exceptions: print e
class ProcessExecutor(object): def __init__(self): self.processes = [] self.q = Queue() def wait_until_finished(self): for _process in self.processes: _process.join() self.q.close() self.q.join_thread() def execute(self, fn, *args, **kwargs): promise = Promise() self.q.put([promise, fn, args, kwargs], False) _process = Process(target=queue_process, args=(self.q)) _process.start() self.processes.append(_process) return promise
def split(data, processes): ''' Fork off a number of processes. The data processing will be split among those processes. The data goes into a queue. Processes pick of data from the queue until the queue is exhausted. processes is the number of processes to start. ''' global _pid, _qout, _processes _processes = processes def forker(): for i in range(processes): process = os.fork() if process != 0: return i if os.fork() == 0: return FeederThread else: return ConsumerThread qin = Queue(maxsize=1000) _pid = forker() # Feeder process if _pid == FeederThread: for d in data: qin.put(d, block=True) for d in range(processes): qin.put(EndOfQueue, block=True) qin.close() qin.join_thread() return [] elif _pid == ConsumerThread: return [] else: return iter(lambda: qin.get(block=True), EndOfQueue)
class ProcessHandler(object): def __init__(self, functions): self.processes = [] self.num_of_processes = len(functions) self.functions = functions self.event_q = Queue() def start(self): # create/start processes and event queue for i in range(self.num_of_processes): function = self.functions[i] name = self.functions[i].__name__ print function self.processes.append(Process(target=function, name=name, args=(self.event_q,))) self.processes[i].start() while not self.processes[i].is_alive(): time.sleep(0.01) print self.processes[i] def close(self): for i in range(self.num_of_processes): self.event_q.close() self.event_q.join_thread() self.processes[i].join() print self.processes[i] def watchDog(self): for i in range(self.num_of_processes): if not self.processes[i].is_alive(): print self.processes[i] function = self.functions[i] name = self.functions[i].__name__ q = self.event_q self.processes[i] = Process(target=function, name=name, args=(q,)) self.processes[i].start() while not self.processes[i].is_alive(): time.sleep(0.1) print self.processes[i]
class Worker: """This class is used for poller and reactionner to work. The worker is a process launch by theses process and read Message in a Queue (self.s) (slave) They launch the Check and then send the result in the Queue self.m (master) they can die if they do not do anything (param timeout) """ id = 0 # None _process = None _mortal = None _idletime = None _timeout = None _c = None def __init__(self, id, s, returns_queue, processes_by_worker, mortal=True, timeout=300, max_plugins_output_length=8192, target=None, loaded_into='unknown', http_daemon=None): self.id = self.__class__.id self.__class__.id += 1 self._mortal = mortal self._idletime = 0 self._timeout = timeout self.s = None self.processes_by_worker = processes_by_worker self._c = Queue() # Private Control queue for the Worker # By default, take our own code if target is None: target = self.work self._process = Process(target=target, args=(s, returns_queue, self._c)) self.returns_queue = returns_queue self.max_plugins_output_length = max_plugins_output_length self.i_am_dying = False # Keep a trace where the worker is launch from (poller or reactionner?) self.loaded_into = loaded_into if os.name != 'nt': self.http_daemon = http_daemon else: #windows forker do not like pickle http/lock self.http_daemon = None def is_mortal(self): return self._mortal def start(self): self._process.start() # Kill the background process # AND close correctly the queues (input and output) # each queue got a thread, so close it too.... def terminate(self): # We can just terminate process, not threads if not is_android: self._process.terminate() # Is we are with a Manager() way # there should be not such functions if hasattr(self._c, 'close'): self._c.close() self._c.join_thread() if hasattr(self.s, 'close'): self.s.close() self.s.join_thread() def join(self, timeout=None): self._process.join(timeout) def is_alive(self): return self._process.is_alive() def is_killable(self): return self._mortal and self._idletime > self._timeout def add_idletime(self, time): self._idletime = self._idletime + time def reset_idle(self): self._idletime = 0 def send_message(self, msg): self._c.put(msg) # A zombie is immortal, so kill not be kill anymore def set_zombie(self): self._mortal = False # Get new checks if less than nb_checks_max # If no new checks got and no check in queue, # sleep for 1 sec # REF: doc/shinken-action-queues.png (3) def get_new_checks(self): try: while(len(self.checks) < self.processes_by_worker): #print "I", self.id, "wait for a message" msg = self.s.get(block=False) if msg is not None: self.checks.append(msg.get_data()) #print "I", self.id, "I've got a message!" except Empty, exp: if len(self.checks) == 0: self._idletime = self._idletime + 1 time.sleep(1) # Maybe the Queue() is not available, if so, just return # get back to work :) except IOError, exp: return
class TestSet(object): "Manage a set of test" def __init__(self, base_dir): """Initalise a test set @base_dir: base directory for tests """ # Parse arguments self.base_dir = base_dir # Init internals self.task_done_cb = lambda tst, err: None # On task done callback self.task_new_cb = lambda tst: None # On new task callback self.todo_queue = Queue() # Tasks to do self.message_queue = Queue() # Messages with workers self.tests = [] # Tests to run self.tests_done = [] # Tasks done self.cpu_c = cpu_count() # CPUs available self.errorcode = 0 # Non-zero if a test failed self.additional_args = [] # Arguments to always add def __add__(self, test): "Same as TestSet.add" self.add(test) return self def add(self, test): "Add a test instance to the current test set" if not isinstance(test, Test): raise ValueError("%s is not a valid test instance" % (repr(test))) self.tests.append(test) def set_cpu_numbers(self, cpu_c): """Set the number of cpu to use @cpu_c: Number of CPU to use (default is maximum) """ self.cpu_c = cpu_c def set_callback(self, task_done=None, task_new=None): """Set callbacks for task information retrieval @task_done: function(Test, Error message) @task_new: function(Test) """ if task_done: self.task_done_cb = task_done if task_new: self.task_new_cb = task_new def _add_tasks(self): "Add tests to do, regarding to dependencies" for test in self.tests: # Check dependencies launchable = True for dependency in test.depends: if dependency not in self.tests_done: launchable = False break if launchable: # Add task self.tests.remove(test) self.todo_queue.put(test) if len(self.tests) == 0: # Poison pills for _ in xrange(self.cpu_c): self.todo_queue.put(None) # All tasks done if len(self.tests_done) == self.init_tests_number: self.message_queue.put(MessageClose()) def _messages_handler(self): "Manage message between Master and Workers" # Main loop while True: message = self.message_queue.get() if isinstance(message, MessageClose): # Poison pill break elif isinstance(message, MessageTaskNew): # A task begins self.task_new_cb(message.task) elif isinstance(message, MessageTaskDone): # A task has been done self.tests_done.append(message.task) self._add_tasks() self.task_done_cb(message.task, message.error) if message.error is not None: self.errorcode = -1 else: raise ValueError("Unknown message type %s" % type(message)) @staticmethod def worker(todo_queue, message_queue, init_args): """Worker launched in parrallel @todo_queue: task to do @message_queue: communication with Host @init_args: additionnal arguments for command line """ # Main loop while True: # Acquire a task test = todo_queue.get() if test is None: break message_queue.put(MessageTaskNew(test)) # Go to the expected directory current_directory = os.getcwd() os.chdir(test.base_dir) # Launch test testpy = subprocess.Popen(([sys.executable] + init_args + test.command_line), stdout=subprocess.PIPE, stderr=subprocess.PIPE) outputs = testpy.communicate() # Check result error = None if testpy.returncode != 0: error = outputs[1] # Restore directory os.chdir(current_directory) # Report task finish message_queue.put(MessageTaskDone(test, error)) @staticmethod def fast_unify(seq, idfun=None): """Order preserving unifying list function @seq: list to unify @idfun: marker function (default is identity) """ if idfun is None: idfun = lambda x: x seen = {} result = [] for item in seq: marker = idfun(item) if marker in seen: continue seen[marker] = 1 result.append(item) return result def _clean(self): "Remove produced files" # Build the list of products products = [] current_directory = os.getcwd() for test in self.tests_done: for product in test.products: # Get the full product path products.append(os.path.join(current_directory, test.base_dir, product)) # Unify the list and remove products for product in TestSet.fast_unify(products): try: os.remove(product) except OSError: print "Cleanning error: Unable to remove %s" % product def add_additionnal_args(self, args): """Add arguments to used on the test command line @args: list of str """ self.additional_args += args def run(self): "Launch tests" # Go in the right directory self.current_directory = os.getcwd() os.chdir(self.base_dir) # Launch workers processes = [] for _ in xrange(self.cpu_c): p = Process(target=TestSet.worker, args=(self.todo_queue, self.message_queue, self.additional_args)) processes.append(p) p.start() # Add initial tasks self.init_tests_number = len(self.tests) # Initial tasks self._add_tasks() # Handle messages self._messages_handler() # Close queue and join processes self.todo_queue.close() self.todo_queue.join_thread() self.message_queue.close() self.message_queue.join_thread() for p in processes: p.join() def end(self, clean=True): """End a testset run @clean: (optional) if set, remove tests products PRE: run() """ # Clean if clean: self._clean() # Restore directory os.chdir(self.current_directory) def tests_passed(self): "Return a non zero value if at least one test failed" return self.errorcode def filter_tags(self, include_tags=None, exclude_tags=None): """Filter tests by tags @include_tags: list of tags' name (whitelist) @exclude_tags: list of tags' name (blacklist) @include_tags and @exclude_tags cannot be used together""" if include_tags and exclude_tags: raise ValueError("Include and Exclude cannot be used together") new_testset_include = [] new_testset_exclude = list(self.tests) # Update include and exclude lists for index, test in enumerate(self.tests): for tag in test.tags: if exclude_tags and tag in exclude_tags: new_testset_exclude.remove(test) break if include_tags and tag in include_tags: new_testset_include.append(test) break # Update testset list if include_tags: self.tests = new_testset_include elif exclude_tags: self.tests = new_testset_exclude
# Prepare multiprocess cpu_c = cpu_count() queue = Queue() processes = [] for _ in xrange(cpu_c): p = Process(target=do_test, args=(args.filename, queue, machine, abicls, tests, map_addr, args.quiet, args.timeout, args.jitter, args.verbose)) processes.append(p) p.start() # Add tasks for address in addresses: queue.put(address) # Add poison pill for _ in xrange(cpu_c): queue.put(None) # Get results queue.close() queue.join_thread() for p in processes: p.join() if not queue.empty(): print("An error occured: queue is not empty")
class Worker: id = 0#None _process = None _mortal = None _idletime = None _timeout = None _c = None def __init__(self, id, s, returns_queue, processes_by_worker, mortal=True, timeout=300, max_plugins_output_length=8192, target=None): self.id = self.__class__.id self.__class__.id += 1 self._mortal = mortal self._idletime = 0 self._timeout = timeout self.processes_by_worker = processes_by_worker self.input_queue = s self._c = Queue() # Private Control queue for the Worker # By default, take our own code if target is None: target=self.work self._process = Process(target=target, args=(s, returns_queue, self._c)) self.returns_queue = returns_queue self.max_plugins_output_length = max_plugins_output_length self.i_am_dying = False def is_mortal(self): return self._mortal def start(self): self._process.start() # Kill the background process # AND close correctly the queues (input and output) # each queue got a thread, so close it too.... def terminate(self): # We can just terminate process, not threads if not is_android: self._process.terminate() # Is we are with a Manager() way # there should be not such functions if hasattr(self._c, 'close'): self._c.close() self._c.join_thread() if hasattr(self.input_queue, 'close'): self.input_queue.close() self.input_queue.join_thread() def join(self, timeout=None): self._process.join(timeout) def is_alive(self): return self._process.is_alive() def is_killable(self): return self._mortal and self._idletime > self._timeout def add_idletime(self, time): self._idletime = self._idletime + time def reset_idle(self): self._idletime = 0 def send_message(self, msg): self._c.put(msg) # A zombie is immortal, so kill not be kill anymore def set_zombie(self): self._mortal = False # Get new checks if less than nb_checks_max # If no new checks got and no check in queue, # sleep for 1 sec # REF: doc/shinken-action-queues.png (3) def get_new_checks(self): try: while(len(self.checks) < self.processes_by_worker): #print "I", self.id, "wait for a message" msg = self.s.get(block=False) if msg is not None: self.checks.append(msg.get_data()) #print "I", self.id, "I've got a message!" except Empty , exp: if len(self.checks) == 0: self._idletime = self._idletime + 1 time.sleep(1) # Maybe the Queue() is not available, if so, just return # get back to work :) except IOError, exp: return
glFlush() pygame.display.flip() try: result = queue_locations.get(False, 0.1) locations.extend(result) print "MOTHER > There are %d locations saved ... :)" % (len(locations)) print "MOTHER > last location:\n", pformat(locations[-1]) # current_location = len(locations)-1 no_data = 0 except QEmpty: if not e_startup.is_set(): p_gps_upd.start() e_startup.set() no_data += r_chrono pass chrono_tmp = chrono chrono = pygame.time.get_ticks() / 1000.0 print "MOTHER : closing queue locations" queue_locations.close() print "MOTHER : waiting end of queue" queue_locations.join_thread() print "MOTHER : set stopped event" e_stopped.set() print "MOTHER : waiting for GPSUPDATER to end" p_gps_upd.join() print "MOTHER : quit PyGame" pygame.quit()
class gpib: visaLib = visa.VisaLibrary() delay = 0#command transmit delay values_format = pyvisa.highlevel.single | pyvisa.highlevel.big_endian #this is now a keithley 2400 does binary transfers chunk_size = 102400 #need a slightly bigger transfer buffer than default to be able to transfer a full sample buffer (2500 samples) from a keithley 2400 in one shot def __init__(self,locationString=None,timeout=30,useQueues=False): self.locationString = locationString self.timeout = timeout self.useQueues = useQueues if self.locationString is not None: if self.useQueues: #queue mode #build the queues self.task_queue = Queue() self.done_queue = Queue() #kickoff the worker process self.p = Process(target=self._worker, args=(self.task_queue, self.done_queue)) self.p.start() else:#non-queue mode self.v = visa.instrument(self.locationString,timeout=self.timeout,chunk_size=self.chunk_size,delay=self.delay,values_format=self.values_format) def __del__(self): if self.useQueues: if self.p.is_alive(): self.task_queue.put('STOP') self.p.join() self.task_queue.close() self.done_queue.close() self.task_queue.join_thread() self.done_queue.join_thread() else: if hasattr(self,'v'): self.v.close() def _worker(self, inputQ, outputQ): #local, threadsafe instrument object created here v = visa.instrument(self.locationString,timeout=self.timeout,chunk_size=self.chunk_size,delay=self.delay,values_format=self.values_format) for func, args in iter(inputQ.get, 'STOP'):#queue processing going on here try: toCall = getattr(v,func) ret = toCall(*args)#visa function call occurs here except: ret = None if ret: #don't put None outputs into output queue outputQ.put(ret) print "queue worker closed properly" v.close() inputQ.close() outputQ.close() #make queue'd and non-queued writes look the same to the client def write(self,string): if self.useQueues: self.task_queue.put(('write',(string,))) else: self.v.write(string) #controls remote enable line def controlRen(self,mode): visa.Gpib()._vpp43.gpib_control_ren(mode) def clearInterface(self): self.visaLib.gpib_send_ifc() def findInstruments(self): return visa.get_instruments_list()
def my_consumer(q): """ Consumes some data and works on it In this case, all it does is double the input """ while True: data = q.get() print('data found to be processed: {}'.format(data)) processed = data * 2 print(processed) if data is sentinel: break if __name__ == '__main__': q = Queue() data = [5, 10, 13, -1] process_one = Process(target=creator, args=(data, q)) process_two = Process(target=my_consumer, args=(q,)) process_one.start() process_two.start() q.close() q.join_thread() process_one.join() process_two.join()
def refine_all(G, **kwargs): realignbubbles=[] if kwargs['minsize']==None: kwargs['minsize']=kwargs['minlength'] #detect all bubbles for b in bubbles.bubbles(G): if kwargs['complex']: if b.issimple(): logging.debug("Skipping bubble %s, not complex."%str(b.nodes)) continue if kwargs['simple']: if not b.issimple(): logging.debug("Skipping bubble %s, not simple."%str(b.nodes)) continue if b.minsize<kwargs['minsize']: logging.debug("Skipping bubble %s, smallest allele (%dbp) is smaller than minsize=%d."%(str(b.nodes),b.minsize,kwargs['minsize'])) continue if b.maxsize<kwargs['minmaxsize']: logging.debug("Skipping bubble %s, largest allele (%dbp) is smaller than minmaxsize=%d."%(str(b.nodes),b.maxsize,kwargs['minmaxsize'])) continue if b.maxsize>kwargs['maxsize']: logging.warn("Skipping bubble %s, largest allele (%dbp) is larger than maxsize=%d."%(str(b.nodes),b.maxsize,kwargs['maxsize'])) continue if kwargs['maxcumsize']!=None: if b.cumsize>kwargs['maxcumsize']: logging.warn("Skipping bubble %s, cumulative size %d is larger than maxcumsize=%d."%(str(b.nodes),b.cumsize,kwargs['maxcumsize'])) continue if b.cumsize<kwargs['mincumsize']: logging.info("Skipping bubble %s, cumulative size %d is smaller than mincumsize=%d."%(str(b.nodes),b.cumsize,kwargs['mincumsize'])) continue if len(b.nodes)==3: logging.info("Skipping bubble %s, indel, no point in realigning."%(str(b.nodes))) continue realignbubbles.append(b) distinctbubbles=[] for b1 in realignbubbles: for b2 in realignbubbles: if set(b2.nodes).issuperset(set(b1.nodes)) and not set(b1.nodes)==set(b2.nodes): logging.debug("Skipping bubble %s, because its nested in %s."%(str(b1.nodes),str(b2.nodes))) break else: distinctbubbles.append(b1) logging.info("Realigning a total of %d bubbles"%len(distinctbubbles)) nn=max([node for node in G.nodes() if type(node)==int])+1 if kwargs['nproc']>1: inputq = Queue() outputq = Queue() nworkers=kwargs['nproc'] aworkers=[] for i in range(nworkers): aworkers.append(Process(target=align_worker, args=(inputq,outputq))) for p in aworkers: p.start() for bubble in distinctbubbles: G.node[bubble.source]['aligned']=1 G.node[bubble.sink]['aligned']=1 logging.debug("Submitting realign bubble between <%s> and <%s>, max allele size %dbp (in nodes=%d)."%(bubble.source,bubble.sink,bubble.maxsize,len(bubble.nodes)-2)) bnodes=list(set(bubble.nodes)-set([bubble.source,bubble.sink])) sg=G.subgraph(bnodes).copy() offsets=dict() for sid in G.node[bubble.source]['offsets']: offsets[sid]=G.node[bubble.source]['offsets'][sid]+len(G.node[bubble.source]['seq']) sourcesamples=set(G.node[bubble.source]['offsets'].keys()) sinksamples=set(G.node[bubble.sink]['offsets'].keys()) paths=sourcesamples.intersection(sinksamples) inputq.put((sg,bubble,offsets,paths,kwargs)) for i in range(nworkers): inputq.put(-1) graph_worker(G,nn,outputq,nworkers) inputq.close() inputq.join_thread() outputq.close() for p in aworkers: p.join() else: for bubble in distinctbubbles: G.node[bubble.source]['aligned']=1 G.node[bubble.sink]['aligned']=1 bnodes=list(set(bubble.nodes)-set([bubble.source,bubble.sink])) sg=G.subgraph(bnodes) offsets=dict() for sid in G.node[bubble.source]['offsets']: offsets[sid]=G.node[bubble.source]['offsets'][sid]+len(G.node[bubble.source]['seq']) sourcesamples=set(G.node[bubble.source]['offsets'].keys()) sinksamples=set(G.node[bubble.sink]['offsets'].keys()) paths=sourcesamples.intersection(sinksamples) res=refine_bubble(sg,bubble,offsets,paths, **kwargs) if res==None: continue else: bubble,ng,path2start,path2end=res G,nn=replace_bubble(G,bubble,ng,path2start,path2end,nn) return G
from clientAplication import Aplication from clientTransport import TCPClient, UDPClient from teste import Work lock = Lock() # app = Aplication() jobs = Queue() udp = Process(target=UDPClient, args=("HTTPRequest.txt", jobs)) udp.start() jobs.put(Aplication()) jobs.wait() jobs.close() jobs.join_thread() app.join() """ app = Process(target=Aplication, args=(jobs,)) app.start() jobs.put(UDPClient("HTTPRequest.txt")) #wait first process to finish jobs.close() jobs.join_thread() app.join() """
class MPlayer(Popen): arg_types = { 'String':(str, '"%s"'), 'Float':(float, '%f'), 'Integer':(int, '%d'), } @property def state(self): return self._state.value def __init__(self, args=None, path=None): self.path = path or 'mplayer' self.args = args or [] self.get_cmds() self.run() def __getattr__(self, attr): if attr in self.cmds: return lambda *args, **kargs: self.send_cmd(attr, args, kargs) raise AttributeError('MPlayer does not respond to: %s' % attr) def run(self): Popen.__init__(self, [self.path, '-idle', '-slave', '-quiet']+self.args, stdin=PIPE, stdout=PIPE, stderr=PIPE) self.manager = Manager() self.defaults = self.manager.dict() self._state = Value('i') self._state.value = STOPPED self.notifier = SelectQueue() self.calls = SelectQueue() self.results = Queue() self.ioworker = IOWorker(self.stdin, self.stdout, self._state, self.notifier, self.calls, self.results) self.ioworker.start() def kill(self): if self.notifier: self.notifier.close() self.notifier.join_thread() if self.results: self.results.close() self.results.join_thread() if self.ioworker: self.ioworker.terminate() if self.poll() is None: self.terminate() def restart(self): self.kill() self.run() def get_cmds(self): self.cmds = {} output = Popen([self.path, '-input', 'cmdlist'], stdout=PIPE) for line in output.stdout: match = re.search(b'^(\w+)\s+(.*)', line) cmd, args = [item.decode('utf-8') for item in match.groups()] self.cmds[cmd] = args.split(' ') if args else [] def process_args(self, cmd, args): try: for i in range(len(args)): arg_type = self.cmds[cmd][i].strip('[]') arg_func, arg_mask = self.arg_types[arg_type] yield arg_mask % arg_func(args[i]) except (IndexError, ValueError): raise ValueError('%s expects arguments of format %r' % (cmd, ' '.join(str(x) for x in self.cmds[cmd]))) def send_cmd(self, cmd, args, kargs): if self.poll(): raise MPlayerNotRunning('MPlayer instance not running.') prefix = kargs.get('prefix', self.defaults.get('prefix', '')) default = kargs.get('default', self.defaults.get('default')) cmd = '%s %s' % (cmd, ' '.join(arg for arg in self.process_args(cmd, args))) self.calls.put([prefix, cmd, default]) return self.results.get()
class TestSet(object): "Manage a set of test" def __init__(self, base_dir): """Initalise a test set @base_dir: base directory for tests """ # Parse arguments self.base_dir = base_dir # Init internals self.task_done_cb = lambda tst, err: None # On task done callback self.task_new_cb = lambda tst: None # On new task callback self.todo_queue = Queue() # Tasks to do self.message_queue = Queue() # Messages with workers self.tests = [] # Tests to run self.tests_done = [] # Tasks done self.cpu_c = cpu_count() # CPUs available self.errorcode = 0 # Non-zero if a test failed self.additional_args = [] # Arguments to always add def __add__(self, test): "Same as TestSet.add" self.add(test) return self def add(self, test): "Add a test instance to the current test set" if not isinstance(test, Test): raise ValueError("%s is not a valid test instance" % (repr(test))) self.tests.append(test) def set_cpu_numbers(self, cpu_c): """Set the number of cpu to use @cpu_c: Number of CPU to use (default is maximum) """ self.cpu_c = cpu_c def set_callback(self, task_done=None, task_new=None): """Set callbacks for task information retrieval @task_done: function(Test, Error message) @task_new: function(Test) """ if task_done: self.task_done_cb = task_done if task_new: self.task_new_cb = task_new def add_tasks(self): "Add tests to do, regarding to dependencies" for test in self.tests: # Check dependencies launchable = True for dependency in test.depends: if dependency not in self.tests_done: launchable = False break if launchable: # Add task self.tests.remove(test) self.todo_queue.put(test) if len(self.tests) == 0: # Poison pills for _ in xrange(self.cpu_c): self.todo_queue.put(None) # All tasks done if len(self.tests_done) == self.init_tests_number: self.message_queue.put(MessageClose()) def messages_handler(self): "Manage message between Master and Workers" # Main loop while True: message = self.message_queue.get() if isinstance(message, MessageClose): # Poison pill break elif isinstance(message, MessageTaskNew): # A task begins self.task_new_cb(message.task) elif isinstance(message, MessageTaskDone): # A task has been done self.tests_done.append(message.task) self.add_tasks() self.task_done_cb(message.task, message.error) if message.error is not None: self.errorcode = -1 else: raise ValueError("Unknown message type %s" % type(message)) @staticmethod def worker(todo_queue, message_queue, init_args): """Worker launched in parrallel @todo_queue: task to do @message_queue: communication with Host @init_args: additionnal arguments for command line """ # Main loop while True: # Acquire a task test = todo_queue.get() if test is None: break message_queue.put(MessageTaskNew(test)) # Go to the expected directory current_directory = os.getcwd() os.chdir(test.base_dir) # Launch test testpy = subprocess.Popen(["python"] + init_args + test.command_line, stdout=subprocess.PIPE, stderr=subprocess.PIPE) outputs = testpy.communicate() # Check result error = None if testpy.returncode != 0: error = outputs[1] # Restore directory os.chdir(current_directory) # Report task finish message_queue.put(MessageTaskDone(test, error)) def clean(self): "Remove produced files" for test in self.tests_done: # Go to the expected directory current_directory = os.getcwd() os.chdir(test.base_dir) # Remove files for product in test.products: try: os.remove(product) except OSError: print "Cleanning error: Unable to remove %s" % product # Restore directory os.chdir(current_directory) def add_additionnal_args(self, args): """Add arguments to used on the test command line @args: list of str """ self.add_additionnal_args += args def run(self): "Launch tests" # Go in the right directory current_directory = os.getcwd() os.chdir(self.base_dir) # Launch workers processes = [] for _ in xrange(self.cpu_c): p = Process(target=TestSet.worker, args=(self.todo_queue, self.message_queue, self.additional_args)) processes.append(p) p.start() # Add initial tasks self.init_tests_number = len(self.tests) # Initial tasks self.add_tasks() # Handle messages self.messages_handler() # Close queue and join processes self.todo_queue.close() self.todo_queue.join_thread() self.message_queue.close() self.message_queue.join_thread() for p in processes: p.join() # Clean self.clean() # Restore directory os.chdir(current_directory) def tests_passed(self): "Return a non zero value if at least one test failed" return self.errorcode
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--server', default=DEFAULT_SERVER, help=u'Elasticsearch hostname or IP (default {0})'.format(DEFAULT_SERVER)) parser.add_argument('--port', default=DEFAULT_PORT, help=u'Elasticsearch port (default {0})'.format(DEFAULT_PORT)) parser.add_argument('--scanfile', help=u'Path to umich scan file you are ingesting. ' u'Please make sure to decompress it') parser.add_argument('--initial', help=u'If this is the first file you are importing please use this flag', action='store_true') args = parser.parse_args(argv[1:]) if args.scanfile is None: logger.error("Please include a scanfile") sys.exit(1) workers = cpu_count() process_hosts_queue = Queue(maxsize=20000) process_certs_queue = Queue(maxsize=20000) for w in xrange(workers/2): # Establish elasticsearch connection for each process es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=30) p = Process(target=process_hosts, args=(process_hosts_queue, es, args.initial)) p.daemon = True p.start() for w in xrange(workers/2): # Establish elasticsearch connection for each process es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=30) p = Process(target=process_scan_certs, args=(process_certs_queue, es)) p.daemon = True p.start() logger.warning("Starting processing of {file} at {date}".format(file=args.scanfile, date=datetime.now())) # This is the bottle neck of the process but it works for now parse_scanfile(args.scanfile, process_hosts_queue, process_certs_queue) # Once all the json lines have been put onto the queue. Add DONE so the queue workers know when to quit. for w in xrange(workers): process_hosts_queue.put("DONE") process_certs_queue.put("DONE") # Close out the queue we are done process_hosts_queue.close() process_hosts_queue.join_thread() process_certs_queue.close() process_certs_queue.join_thread() # this is kinda dirty but without looking up everything at insert time (slow) I don't know of a better way to do # this based on the number of documents we will have refresh_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=30) # construct an elasticsearch query where the filter is looking for any entry that is missing the field first_seen q = {'size': 500, "query": {"match_all": {}}, "filter": {"missing": {"field": "first_seen"}}} new_updates = refresh_es.search(index='passive-ssl-hosts-umich', body=q) logger.warning("Numer of hosts to update is {count}".format(count=new_updates['hits']['total'])) # Scan across all the documents missing the first_seen field and bulk update them missing_first_seen = scan(refresh_es, query=q, scroll='30m', index='passive-ssl-hosts-umich') bulk_miss = [] for miss in missing_first_seen: last_seen = miss['_source']['last_seen'] first_seen = last_seen action = {"_op_type": "update", "_index": "passive-ssl-hosts-umich", "_type": "host", "_id": miss['_id'], "doc": {'first_seen': first_seen}} bulk_miss.append(action) if len(bulk_miss) == 500: bulk(refresh_es, bulk_miss) bulk_miss = [] # Get the remaining ones that are less than 000 and the loop has ended bulk(refresh_es, bulk_miss) logger.warning("{file} import finished at {date}".format(file=args.scanfile, date=datetime.now())) # Now we should optimize each index to max num segments of 1 to help with searching/sizing and just over all # es happiness logger.warning("Optimizing index: {index} at {date}".format(index='passive-ssl-hosts-umich', date=datetime.now())) refresh_es.indices.optimize(index='passive-ssl-hosts-umich', max_num_segments=1, request_timeout=7500) logger.warning("Optimizing index: {index} at {date}".format(index='passive-ssl-certs-umich', date=datetime.now())) refresh_es.indices.optimize(index='passive-ssl-certs-umich', max_num_segments=1, request_timeout=7500)
class SynergeticPool(Process): """Synergetic Process Pool: """ def __init__(self, synergetic_servers=None, local_workers=1, syn_listener_port=41000): # Enable Class Method Pickling copy_reg.pickle(types.MethodType, _reduce_method) # Enable Descriptor Method Pickling copy_reg.pickle(types.MemberDescriptorType, _reduce_method_descriptor) # Dictionary of the synergetic servers' IP Addresses or Names with their connections self.__syn_servs = dict() # List of available Synergetic Processes self.__syn_pcss = list() # List of incomplete tasks recored self.__incomp_tsks = dict() # # # Not on USE for NOW # Start the Listener that is expecting new-coming synergetic-servers of synergetic-processes self.__start_synergetic_listener(listener_port=syn_listener_port) # Start the Synergetic-Pool's functionality self.__start_pool(synergetic_servers, local_workers_num=local_workers) # Start the Synergetic feeder that feeds the remote servers with Tasks # self.__start_synergetic_feeder() # Start the Synergetic Receiver for getting the results of the remote processes # self.__start_synergetic_recver() def register_mod(self, mod_list): __regstr_mods = dict() for mod in mod_list: try: fobj = open(mod + ".pyc", 'rb') except Exception as e: print("Synergeticprocessing.Pool--> Module registration failed: %s" % e) else: mod_bytecod = fobj.read() finally: fobj.close() # Registering mod. __regstr_mods[mod] = mod_bytecod for serv, conn in self.__syn_servs.items(): if conn: try: conn.send(('MODULES', __regstr_mods)) print 'MSG SND to ', serv except Exception as e: raise Exception('Module Registration Error: %s' % e) ret = self.__return_queue.get() while ret != 'MODULES-READY': self.__return_queue.put(ret) ret = self.__return_queue.get() print('Modules Ready @ SynergeticServer: %s' % serv) def __start_pool(self, synergetic_servers, local_workers_num): # Initialise the Queues if synergetic_servers: syn_servs_num = len(synergetic_servers) self.__task_queue = JoinableQueue() # syn_servs_num + local_workers_num) self.__return_queue = Queue(syn_servs_num + local_workers_num) # self.__start_local_pool(local_workers_num) # self.__start_serv_pool(synergetic_servers) # Start the Synergetic feeder that feeds the remote servers with Tasks self.__start_synergetic_feeder() # Start the Synergetic Receiver for getting the results of the remote processes self.__start_synergetic_recver() else: self.__task_queue = JoinableQueue() # local_workers_num self.__return_queue = Queue(local_workers_num) # self.__start_local_pool(local_workers_num) # self.__task_queue = JoinableQueue(syn_servs_num + local_workers_num) # self.__return_queue = Queue(syn_servs_num + local_workers_num) # # # self.__start_local_pool(local_workers_num) # # # self.__start_serv_pool(synergetic_servers) def __start_synergetic_listener(self, listener_port): listener = Process(target=self.__synergetic_serv_listener, args=(listener_port,)) listener.daemon = True listener.start() def __synergetic_serv_listener(self, listener_port): serv = Listener(('', listener_port), authkey='123456') while True: conn = serv.accept() try: server, port, authkey = conn.recv() except EOFError: recptn_failed = True if recptn_failed is False: conn.send('WELLCOME') conn.close() def __start_synergetic_feeder(self): feeder = Process(target=self.__feed_remote_syncss) feeder.daemon = True feeder.start() def __feed_remote_syncss(self): while True: for serv_addr, conn in self.__syn_servs.items(): if conn: Task = self.__task_queue.get() self.__task_queue.task_done() # Keep Task Until is completed # if Task[0] != 'MODULES': # (task_id, func, args, kwargs) = Task # self.__incomp_tsks[serv_addr]= { task_id : (func, args, kwargs) } try: conn.send(Task) except EOFError: self.__syn_servs[serv_addr] = None def __start_synergetic_recver(self): recver = Process(target=self.__recv_remote_syncss) recver.daemon = True recver.start() def __recv_remote_syncss(self): while True: for serv_addr, conn in self.__syn_servs.items(): if conn: try: return_msg = conn.recv() except EOFError: self.__syn_servs[serv_addr] = None else: self.__return_queue.put(return_msg) if serv_addr in self.__incomp_tsks and return_msg != 'MODULES-READY': del self.__incomp_tsks[serv_addr][return_msg[0]] # elif self.__incomp_tsks[serv_addr]: # Tasks = self.__incomp_tsks[serv_addr] # for task_id, (func, args, kwargs) in Tasks.items(): # print "Incomplete task putting back to Queue" # self.__task_queue.put((task_id, func, args, kwargs)) # del self._incomp_tsks[serv_addr][task_id] # self.__task_queue.task_done() def __synergetic_serv_connection(self, serv, port, auth): try: conn = Client((serv, port), authkey=str(auth)) except: conn = None return conn def __start_local_pool(self, local_worker_num=1): for i in range(local_worker_num): self.__syn_pcss.append(SynergeticProcess(self.__task_queue, self.__return_queue)) for syn_p in self.__syn_pcss: syn_p.daemonic = True syn_p.start() def __start_serv_pool(self, synergetic_servers): for serv, (port, auth) in synergetic_servers.items(): print serv, port, auth conn = self.__synergetic_serv_connection(serv, port, auth) if conn: self.__syn_servs[serv] = conn else: # Maybe this will be DEPRICATED self.__syn_servs[serv] = None def __dispatch(self, func, *args, **kwargs): task_id = str(random.randrange(1, 100000000)) task = (task_id, func, args, kwargs) self.__task_queue.put(task) return task_id def dispatch(self, func, *args, **kwargs): task_id = self.__dispatch(func, *args, **kwargs) return ResaultIterator(self.__return_queue, [task_id]) def imap(self, func, iterable=None, chank=0, callback=None): task_ids = list() if chank == 0: # self.__task_queue = JoinableQueue(len(iterable)) # self.__return_queue = Queue(len(iterable)) count = 0 for itr_item in iterable: # task_ids = list() # resault = ResaultIterator(self.__return_queue, task_ids) # resault._job(self.__dispatch, func, iterable)) count += 1 print "dispatching", count task_id = self.__dispatch(func, itr_item) task_ids.append(task_id) # return resault else: chank_size = len(iterable) / chank chank_res_num = len(iterable) % chank for i in range(chank): start_pntr = i * chank_size end_pntr = (i+1) * chank_size itr_chank = iterable[start_pntr:end_pntr] task_id = self.__dispatch(func, itr_chank) task_ids.append(task_id) if chank_res_num: itr_chank = iterable[end_pntr:(end_pntr + chank_res_num)] task_id = self.__dispatch(func, itr_chank) task_ids.append(task_id) print "Returning" return ResaultIterator(self.__return_queue, task_ids) def map(self, func, iterable=None, chank=1, callback=None): ret = list() iter_ret = self.imap(func, iterable, chank, callback) for ret_item in iter_ret: ret.append(ret_item) return ret def join_all(self, timeout=None): for serv, conn in self.__syn_servs.items(): if conn: conn.close() print("Connection to Synergetic-server:%s CLOSED" % serv) self.__task_queue.put((None, None, None, None)) self.__task_queue.task_done() self.__task_queue.join() self.__return_queue.close() self.__return_queue.join_thread() @property def remote_prcss_num(self): return len(self.__syn_pcss) @property def local_prcss_num(self): return len(self.__syn_servs)
class QueuedWriter(object): """ A queued multiprocess writter. """ def __init__(self, func): """ Queue_Writer(func) -> Fill a queue and call func with data as it is able to use it. """ self._func = func # Queue to fill with data to be written. self._data_queue = Queue() # Queue to get the return value from the write function. self._return_queue = Queue() # Create the writer process. self._writer_p = Process(target=_queue_writer, args=(func, self._data_queue, self._return_queue)) self._writer_p.start() def __repr__(self): """ __repr__ -> Returns a python expression to recreate this instance. """ repr_str = "%(_func)s" % self.__dict__ return '%s(%s)' % (self.__class__.__name__, repr_str) def write(self, data): """ Send more data down the queue to the processing function. """ self._data_queue.put(data) return len(data) __call__ = write def close(self): """ Close the queue and writer process. """ # Send EOF to end the writer process. self._data_queue.put('EOF') # Close the queue. self._data_queue.close() # Wait for the queue buffer to empty. self._data_queue.join_thread() # Get the return value from the writer process. ret_val = self._return_queue.get() # Close the return queue. self._return_queue.close() # Wait for the writer process to exit. self._writer_p.join() # Return the writer result. return ret_val def __enter__(self): """ Provides the ability to use pythons with statement. """ try: return self except Exception as err: print(err) return None def __exit__(self, exc_type, exc_value, traceback): """ Close the file when finished. """ try: return self.close() or not bool(exc_type) except Exception as err: print(err) return False
class Worker: """This class is used for poller and reactionner to work. The worker is a process launch by theses process and read Message in a Queue (self.s) (slave) They launch the Check and then send the result in the Queue self.m (master) they can die if they do not do anything (param timeout) """ _id = 0 # None _process = None _mortal = None _idletime = None _timeout = None _control_q = None def __init__(self, _id, slave_q, returns_queue, processes_by_worker, mortal=True, timeout=300, max_plugins_output_length=8192, target=None, loaded_into='unknown', http_daemon=None): self._id = self.__class__._id self.__class__._id += 1 self._mortal = mortal self._idletime = 0 self._timeout = timeout self.slave_q = None self.processes_by_worker = processes_by_worker self._control_q = Queue() # Private Control queue for the Worker # By default, take our own code if target is None: target = self.work self._process = Process(target=target, args=(slave_q, returns_queue, self._control_q)) self.returns_queue = returns_queue self.max_plugins_output_length = max_plugins_output_length self.i_am_dying = False # Keep a trace where the worker is launch from (poller or reactionner?) self.loaded_into = loaded_into if os.name != 'nt': self.http_daemon = http_daemon else: # windows forker do not like pickle http/lock self.http_daemon = None def is_mortal(self): """ Accessor to _mortal attribute :return: A boolean indicating if the worker is mortal or not. :rtype: bool """ return self._mortal def start(self): """ Start the worker. Wrapper for calling start method of the process attribute :return: None """ self._process.start() def terminate(self): """ Wrapper for calling terminate method of the process attribute Also close queues (input and output) and terminate queues thread :return: None """ # We can just terminate process, not threads self._process.terminate() # Is we are with a Manager() way # there should be not such functions if hasattr(self._control_q, 'close'): self._control_q.close() self._control_q.join_thread() if hasattr(self.slave_q, 'close'): self.slave_q.close() self.slave_q.join_thread() def join(self, timeout=None): """ Wrapper for calling join method of the process attribute :param timeout: time to wait for the process to terminate :type timeout: int :return: None """ self._process.join(timeout) def is_alive(self): """ Wrapper for calling is_alive method of the process attribute :return: A boolean indicating if the process is alive :rtype: bool """ return self._process.is_alive() def is_killable(self): """ Determine whether a process is killable : * process is mortal * idletime > timeout :return: a boolean indicating if it is killable :rtype: bool """ return self._mortal and self._idletime > self._timeout def add_idletime(self, time): """ Increment idletime :param time: time to increment in seconds :type time: int :return: None """ self._idletime += time def reset_idle(self): """ Reset idletime (set to 0) :return: None """ self._idletime = 0 def send_message(self, msg): """ Wrapper for calling put method of the _control_q attribute :param msg: the message to put in queue :type msg: str :return: None """ self._control_q.put(msg) def set_zombie(self): """ Set the process as zombie (mortal to False) :return:None """ self._mortal = False def get_new_checks(self): """ Get new checks if less than nb_checks_max If no new checks got and no check in queue, sleep for 1 sec REF: doc/alignak-action-queues.png (3) :return: None """ try: while len(self.checks) < self.processes_by_worker: # print "I", self._id, "wait for a message" msg = self.slave_q.get(block=False) if msg is not None: self.checks.append(msg.get_data()) # print "I", self._id, "I've got a message!" except Empty, exp: if len(self.checks) == 0: self._idletime += 1 time.sleep(1) # Maybe the Queue() is not available, if so, just return # get back to work :) except IOError, exp: return
class Sentinel(object): def __init__(self, stop_event, start_event, list_key=Conf.Q_LIST, timeout=Conf.TIMEOUT, start=True): # Make sure we catch signals for the pool signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_DFL) self.pid = current_process().pid self.parent_pid = os.getppid() self.name = current_process().name self.list_key = list_key self.r = redis_client self.reincarnations = 0 self.tob = timezone.now() self.stop_event = stop_event self.start_event = start_event self.pool_size = Conf.WORKERS self.pool = [] self.timeout = timeout self.task_queue = Queue() self.result_queue = Queue() self.event_out = Event() self.monitor = Process() self.pusher = Process() if start: self.start() def start(self): self.spawn_cluster() self.guard() def status(self): if not self.start_event.is_set() and not self.stop_event.is_set(): return Conf.STARTING elif self.start_event.is_set() and not self.stop_event.is_set(): if self.result_queue.qsize() == 0 and self.task_queue.qsize() == 0: return Conf.IDLE return Conf.WORKING elif self.stop_event.is_set() and self.start_event.is_set(): if self.monitor.is_alive() or self.pusher.is_alive() or len(self.pool) > 0: return Conf.STOPPING return Conf.STOPPED def spawn_process(self, target, *args): """ :type target: function or class """ # This is just for PyCharm to not crash. Ignore it. if not hasattr(sys.stdin, 'close'): def dummy_close(): pass sys.stdin.close = dummy_close p = Process(target=target, args=args) p.daemon = True if target == worker: p.timer = args[2] self.pool.append(p) p.start() return p def spawn_pusher(self): return self.spawn_process(pusher, self.task_queue, self.event_out, self.list_key, self.r) def spawn_worker(self): self.spawn_process(worker, self.task_queue, self.result_queue, Value('f', -1), self.timeout) def spawn_monitor(self): return self.spawn_process(monitor, self.result_queue) def reincarnate(self, process): """ :param process: the process to reincarnate :type process: Process """ if process == self.monitor: self.monitor = self.spawn_monitor() logger.error(_("reincarnated monitor {} after sudden death").format(process.name)) elif process == self.pusher: self.pusher = self.spawn_pusher() logger.error(_("reincarnated pusher {} after sudden death").format(process.name)) else: self.pool.remove(process) self.spawn_worker() if self.timeout and int(process.timer.value) == 0: # only need to terminate on timeout, otherwise we risk destabilizing the queues process.terminate() logger.warn(_("reincarnated worker {} after timeout").format(process.name)) elif int(process.timer.value) == -2: logger.info(_("recycled worker {}").format(process.name)) else: logger.error(_("reincarnated worker {} after death").format(process.name)) self.reincarnations += 1 def spawn_cluster(self): self.pool = [] Stat(self).save() # spawn worker pool for i in range(self.pool_size): self.spawn_worker() # spawn auxiliary self.monitor = self.spawn_monitor() self.pusher = self.spawn_pusher() # set worker cpu affinity if needed if psutil and Conf.CPU_AFFINITY: set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool]) def guard(self): logger.info(_('{} guarding cluster at {}').format(current_process().name, self.pid)) self.start_event.set() Stat(self).save() logger.info(_('Q Cluster-{} running.').format(self.parent_pid)) scheduler(list_key=self.list_key) counter = 0 cycle = 0.5 # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: # Are you alive? if not p.is_alive() or (self.timeout and p.timer.value == 0): self.reincarnate(p) continue # Decrement timer if work is being done if self.timeout and p.timer.value > 0: p.timer.value -= cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate(self.monitor) # Check Pusher if not self.pusher.is_alive(): self.reincarnate(self.pusher) # Call scheduler once a minute (or so) counter += cycle if counter == 30: counter = 0 scheduler(list_key=self.list_key) # Save current status Stat(self).save() sleep(cycle) self.stop() def stop(self): Stat(self).save() name = current_process().name logger.info('{} stopping cluster processes'.format(name)) # Stopping pusher self.event_out.set() # Wait for it to stop while self.pusher.is_alive(): sleep(0.1) Stat(self).save() # Put poison pills in the queue for _ in range(len(self.pool)): self.task_queue.put('STOP') self.task_queue.close() # wait for the task queue to empty self.task_queue.join_thread() # Wait for all the workers to exit while len(self.pool): for p in self.pool: if not p.is_alive(): self.pool.remove(p) sleep(0.1) Stat(self).save() # Finally stop the monitor self.result_queue.put('STOP') self.result_queue.close() # Wait for the result queue to empty self.result_queue.join_thread() logger.info('{} waiting for the monitor.'.format(name)) # Wait for everything to close or time out count = 0 if not self.timeout: self.timeout = 30 while self.status() == Conf.STOPPING and count < self.timeout * 10: sleep(0.1) Stat(self).save() count += 1 # Final status Stat(self).save()
class RecordTest: """A context manager that records a test to the database. Can also be invoked with record=False as a noop.""" def __init__(self, record=True, reinstall_on_failure=True, description=None, build=None, dut=None, stdout_filter=None, result_id=None, record_finish=True, job_id=None): self.record = record self.description = description self.reinstall_on_failure = reinstall_on_failure self.build = build self.dut = dut self.result_id = result_id self.mdb = self.result_id = self.dut_id = None self.stdout_filter = stdout_filter self.record_queue = self.stream_process = None self.record_finish = record_finish self.failed = False self.job_id = job_id def __enter__(self): """Start recording a test""" try: run(['logger', 'BVT', 'starting', self.full_description()], host=self.dut, timeout=10) except SubprocessError: print 'INFO: unable to mark test log' if not self.record: return self if self.result_id is None: self.mdb = get_autotest() terms = {'test_case':self.description or 'to be determined', 'automation_user': getpwuid(getuid()).pw_gecos.split(',')[0], 'control_pid' : getpid(), 'start_time' : time(), 'development_mode' : 0, 'command_line':abbreviate(' '.join(sys.argv))} if self.dut: dutdoc = self.mdb.duts.find_one({'name':self.dut}) self.dut_id = terms['dut'] = dutdoc['_id'] terms['dut_name'] = dutdoc['name'] if 'development_mode' in dutdoc: terms['development_mode'] = dutdoc['development_mode'] self.result_id = self.mdb.results.save(terms) if self.job_id is not None: self.mdb.jobs.update({'_id':objectid.ObjectId(self.job_id)}, {'$set':{'results_id':self.result_id}}) if self.build is None and self.dut: self.build = get_build(self.dut, timeout=10) self.mdb.results.update({'_id':self.result_id}, {'$set':{'build':self.build}}) if self.dut: self.mdb.duts.update({'_id':terms['dut']}, {'$set': { 'build':self.build, 'control_command_line': abbreviate(' '.join(sys.argv)), 'result_id' : self.result_id}}) if self.stdout_filter: self.record_queue = Queue() self.stream_process = Process( target=service_queue, args=[self.record_queue, self.result_id, self.dut, self.dut_id]) self.stream_process.start() self.stdout_filter.add_callback(self, lambda *x: self.record_queue.put(x)) if self.description: print 'HEADLINE: starting', self.full_description() get_track().updates.save({'result_id':self.result_id, 'action':'new result record'}) return self def __exit__(self, _type, value, traceback): """Finish a test, recording the exception as a test failure, or regarding a test as passed if there is no exception.""" try: if not self.record_finish: return print >>sys.stderr, 'record', self.record dutset = {'last_finish_time':time()} if not self.record: return upd = {'end_time': time(), 'modification_time':time()} if value: # i.e. , if test failed: upd['failure'] = repr(value) upd['exception'] = value.__class__.__name__ if not isinstance(value, KeyboardInterrupt): print 'HEADLINE: exception', upd['exception'], value for clause in format_exception(_type, value, traceback): for line in clause.split('\n'): print 'CRASH:', line else: upd['infrastructure_problem'] = True upd['whiteboard'] = '[infrastructure] test interrupted' if self.reinstall_on_failure: dutset['test_failed'] = True tnext = time() + 300 print 'INFO: test failed, so will reinstall machine at', \ asctime(localtime(tnext)) if self.failed: #some test suite failed upd['failure'] = 'test failed' self.mdb.results.update({'_id':self.result_id}, {'$set':upd}) classify = process_result(self.mdb.results.find_one({'_id':self.result_id})) print 'HEADLINE:', classify, self.full_description() get_track().updates.save({'result_id':self.result_id, 'action':'experiment finished'}) if self.dut_id: self.mdb.duts.update({'_id':self.dut_id}, {'$unset': {'control_pid':1, 'result_id':1, 'control_command_line':1}, '$set': dutset}) if self.build: recount(self.build) if classify == 'infrastructure_problems': pass else: col = 'green' if classify == 'passes' else 'red' finally: if self.record_queue: self.record_queue.put('finish') self.record_queue.close() self.record_queue.join_thread() if self.stream_process: self.stream_process.join() if self.stdout_filter: self.stdout_filter.del_callback(self) def full_description(self): """Work out a full test description""" des = describe_dut(self.dut) if self.dut else '' if self.build: des += ' with ' + self.build if self.result_id: des += ' BVT result ID ' + str(self.result_id) return (self.description if self.description else 'unknown test') + ' on ' + des def set_description(self, description): """Set description (sometimes this is not known at the start of the test""" self.description = description if not self.record: return self.mdb.results.update({'_id':self.result_id}, {'$set':{'test_case':description}}) def set_build(self, build): """Set build (sometimes this is not known at the start of the test""" self.build = build if not self.record: return self.mdb.results.update({'_id':self.result_id}, {'$set':{'build':build}}) # Update the time, result, and failure reason for a step in a test suite. def update_step(self, suite, i, result, reason=''): self.mdb.suiteresults.update({'result_id':self.result_id, 'suite':suite}, {'$set':{'step'+str(i):result, 'step%s-reason'%str(i):reason, 'step%s-end'%str(i):time()}}) # Update the end time, result, and failure reason for a test suite def update_suite(self, suite, result, reason=''): self.mdb.suiteresults.update({'result_id':self.result_id, 'suite':suite}, {'$set':{'result':result, 'finish_time':time(), 'reason':reason}}) # Allocate the initial doc in the suiteresults collection in mongo for this run. def gen_suite_log(self, suite, num_steps): self.mdb.suiteresults.save({'result_id':self.result_id, 'suite':suite, 'steps':num_steps, 'start_time':time()}) # Indicate the start time for one step in a test suite def step_start(self, suite, i): self.mdb.suiteresults.update({'result_id':self.result_id, 'suite':suite}, {'$set':{'step%s-start'%str(i):time()}})