def _init_workers(self): jt.clean() jt.gc() self.index_list = mp.Array('i', self.real_len, lock=False) workers = [] # batch id to worker id self.idmap = mp.Array('i', self.batch_len, lock=False) # global token index self.gid = mp.Value('i', self.batch_len) # global token index condition self.gidc = mp.Condition(self.gid.get_lock()) # number of idle workers self.num_idle = mp.Value('i', 0, lock=False) # number of idle workers condition self.num_idle_c = mp.Condition(self.gid.get_lock()) for i in range(self.num_workers): w = Worker(target=self._worker_main, args=(i, ), buffer_size=self.buffer_size, keep_numpy_array=self.keep_numpy_array) workers.append(w) self.workers = workers self.index_list_numpy = np.ndarray(dtype='int32', shape=self.real_len, buffer=self.index_list)
def test_check_for_downloads_from_md5(): lpath = 'lpath' rfile = azmodels.StorageEntity('cont') rfile._md5 = 'abc' rfile._client = mock.MagicMock() rfile._client.primary_endpoint = 'ep' rfile._name = 'name' rfile._vio = None rfile._size = 256 key = ops.Downloader.create_unique_transfer_operation_id(rfile) d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [ None, (key, lpath, rfile._size, False), ] d._add_to_download_queue = mock.MagicMock() d._all_remote_files_processed = False d._download_terminate = True d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 0 with mock.patch( 'blobxfer.operations.download.Downloader.' 'termination_check_md5', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [ None, (key, lpath, rfile._size, False), ] d._add_to_download_queue = mock.MagicMock() patched_tc.side_effect = [False, False, True] d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 1 with mock.patch( 'blobxfer.operations.download.Downloader.' 'termination_check_md5', new_callable=mock.PropertyMock) as patched_tc: d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._md5_map[key] = rfile d._transfer_set.add(key) d._md5_offload = mock.MagicMock() d._md5_offload.done_cv = multiprocessing.Condition() d._md5_offload.pop_done_queue.side_effect = [None] d._add_to_download_queue = mock.MagicMock() patched_tc.side_effect = [False, True, True] d._check_for_downloads_from_md5() assert d._add_to_download_queue.call_count == 0
def __init__(self): self.ids = multiprocessing.Array('i', 10, lock=False) self.digs = multiprocessing.Array('i', 10, lock=False) self.active = multiprocessing.Array('i', 10, lock=False) self.total_digs = multiprocessing.Value('i', lock=False) self.total_active = multiprocessing.Value('i', lock=False) self.lock = multiprocessing.Lock() self.producer_cond = multiprocessing.Condition(self.lock) self.consumer_cond = multiprocessing.Condition(self.lock)
def __init__(self, maxsize=0): '''initialize the queue''' self.mutex = multiprocessing.Lock() self.not_empty = multiprocessing.Condition(self.mutex) self.not_full = multiprocessing.Condition(self.mutex) self.maxsize = maxsize self._tags = {} # list of refid's for each tag self._queue = {} # the actual queue data self._refcount = { } # how many tags refer to a given refid in the queue self.id_generator = id_generator()
def __init__(self, max_cpu): self.__max_cpu = max_cpu self.__runing = multiprocessing.Value('I', 0, lock=True) self.__close = multiprocessing.Value('I', 0, lock=True) # 进程计数 self.__count = multiprocessing.Value('I', 0, lock=True) # 结束条件变量 self.__finish_cond = multiprocessing.Condition() # 单个进程结束条件变量 self.__count_cond = multiprocessing.Condition() self.__task_que = multiprocessing.Queue()
def test(): manager = multiprocessing.Manager() gc.disable() print '\n\t######## testing Queue.Queue\n' test_queuespeed(threading.Thread, Queue.Queue(), threading.Condition()) print '\n\t######## testing multiprocessing.Queue\n' test_queuespeed(multiprocessing.Process, multiprocessing.Queue(), multiprocessing.Condition()) print '\n\t######## testing Queue managed by server process\n' test_queuespeed(multiprocessing.Process, manager.Queue(), manager.Condition()) print '\n\t######## testing multiprocessing.Pipe\n' test_pipespeed() print print '\n\t######## testing list\n' test_seqspeed(range(10)) print '\n\t######## testing list managed by server process\n' test_seqspeed(manager.list(range(10))) print '\n\t######## testing Array("i", ..., lock=False)\n' test_seqspeed(multiprocessing.Array('i', range(10), lock=False)) print '\n\t######## testing Array("i", ..., lock=True)\n' test_seqspeed(multiprocessing.Array('i', range(10), lock=True)) print print '\n\t######## testing threading.Lock\n' test_lockspeed(threading.Lock()) print '\n\t######## testing threading.RLock\n' test_lockspeed(threading.RLock()) print '\n\t######## testing multiprocessing.Lock\n' test_lockspeed(multiprocessing.Lock()) print '\n\t######## testing multiprocessing.RLock\n' test_lockspeed(multiprocessing.RLock()) print '\n\t######## testing lock managed by server process\n' test_lockspeed(manager.Lock()) print '\n\t######## testing rlock managed by server process\n' test_lockspeed(manager.RLock()) print print '\n\t######## testing threading.Condition\n' test_conditionspeed(threading.Thread, threading.Condition()) print '\n\t######## testing multiprocessing.Condition\n' test_conditionspeed(multiprocessing.Process, multiprocessing.Condition()) print '\n\t######## testing condition managed by a server process\n' test_conditionspeed(multiprocessing.Process, manager.Condition()) gc.enable()
def __init__(self): """ Initialize lock state """ # NOTE: If ever we want an equal access priority instead of giving # access priority to the writers, we could have a global queue FIFO # queue instead of just the pending writers queue. This would move this # from a case-2 RWLock problem to a case-3, ensuring zero starvation of # either lock. # Condition lock on internal variables # noinspection PyUnresolvedReferences self.__cond = multiprocessing.Condition() # Records the ID of the current writer. If this is None, there isn't # currently and entity that has the write lock. #: :type: None or int self.__writer = None # Allows write locks to be reentrant. The write lock is only released # when this hits zero during a releaseWrite self.__writer_count = 0 # Writers will be serviced before readers, recorded by this count. # Writers serviced in order of request as ordered in this list. #: :type: deque of (int, int) self.__pending_writers = deque() # insert linked-list structure here? # current processes holding reader locks. Maps the user's ID to the # number of times it has reentered reading locks. #: :type: dict of (int, int) self.__readers = {}
def __init__(self, is_threaded=True): if is_threaded: self._read_ready = threading.Condition(threading.Lock()) else: self._read_ready = multiprocessing.Condition(multiprocessing.Lock()) self._readers = 0
def _acquire_join_entry(self, i: int, j: int) -> None: self._ensure_join_entry(i, j) if self._join_matrix[i][j] is None: self._join_matrix[i][j] = multiprocessing.Condition() self._join_matrix[i][j].acquire() else: raise RuntimeError('PANIC!')
def test_worker(): top_k_ready = multiprocessing.Condition() with multiprocessing.Manager() as m: top_k = m.dict() count_q = multiprocessing.Queue() vocab_q = multiprocessing.Queue() worker = bpe.Worker(top_k_ready, top_k, count_q, vocab_q, 'aaabb', 100, 2) worker.start() assert vocab_q.get() == {b'a', b'b'} counts = count_q.get() assert counts == {(b'a', b'a'): 2, (b'a', b'b'): 1, (b'b', b'b'): 1} top_k.update( {x[0]: b''.join(x[0]) for x in Counter(counts).most_common(2)}) with top_k_ready: top_k_ready.notify() # Round 2. counts = count_q.get() assert counts == {(b'aa', b'ab'): 1, (b'ab', b'b'): 1} # Finish. top_k.clear() with top_k_ready: top_k_ready.notify() worker.join() assert not worker.is_alive()
def __init__(self, size): self.size = size self.l = mp.Value(ctypes.c_longlong, 0, lock=False) self.r = mp.Value(ctypes.c_longlong, 0, lock=False) self.is_full = mp.Value(ctypes.c_bool, False, lock=False) self.lock = mp.Lock() self.cv = mp.Condition(self.lock)
def init_workers(self): """ Initialize all types of workers and start their worker processes. """ actor_queues = [MpQueue(2 * 1000 * 1000) for _ in range(self.cfg.num_workers)] policy_worker_queues = dict() for policy_id in range(self.cfg.num_policies): policy_worker_queues[policy_id] = [] for i in range(self.cfg.policy_workers_per_policy): policy_worker_queues[policy_id].append(TorchJoinableQueue()) log.info('Initializing learners...') policy_locks = [multiprocessing.Lock() for _ in range(self.cfg.num_policies)] resume_experience_collection_cv = [multiprocessing.Condition() for _ in range(self.cfg.num_policies)] learner_idx = 0 for policy_id in range(self.cfg.num_policies): learner_worker = LearnerWorker( learner_idx, policy_id, self.cfg, self.obs_space, self.action_space, self.report_queue, policy_worker_queues[policy_id], self.traj_buffers, policy_locks[policy_id], resume_experience_collection_cv[policy_id], ) learner_worker.start_process() learner_worker.init() self.learner_workers[policy_id] = learner_worker learner_idx += 1 log.info('Initializing policy workers...') for policy_id in range(self.cfg.num_policies): self.policy_workers[policy_id] = [] policy_queue = MpQueue() self.policy_queues[policy_id] = policy_queue for i in range(self.cfg.policy_workers_per_policy): policy_worker = PolicyWorker( i, policy_id, self.cfg, self.obs_space, self.action_space, self.traj_buffers, policy_queue, actor_queues, self.report_queue, policy_worker_queues[policy_id][i], policy_locks[policy_id], resume_experience_collection_cv[policy_id], ) self.policy_workers[policy_id].append(policy_worker) policy_worker.start_process() log.info('Initializing actors...') # We support actor worker initialization in groups, which can be useful for some envs that # e.g. crash when too many environments are being initialized in parallel. # Currently the limit is not used since it is not required for any envs supported out of the box, # so we parallelize initialization as hard as we can. # If this is required for your environment, perhaps a better solution would be to use global locks, # like FileLock (see doom_gym.py) self.actor_workers = [] max_parallel_init = int(1e9) # might be useful to limit this for some envs worker_indices = list(range(self.cfg.num_workers)) for i in range(0, self.cfg.num_workers, max_parallel_init): workers = self.init_subset(worker_indices[i:i + max_parallel_init], actor_queues) self.actor_workers.extend(workers)
def __init__(self, threshold=0, maxsize=0, sentinel=SENTINEL): """ A Thresholded Queue. It provides the Condition object thesh_cond. Threads waiting on thresh_cond are notified whenever the number of elements (qsize) falls below the specified threshold. Input: threshold threshold condition maxsize maximum number of elements (put blocks if full) """ # Queue is an old style class, we can not use super Queue.Queue.__init__(self, maxsize=maxsize) logger.info('Init queue maxsize: %d, threshold: %d', maxsize, threshold) # the threshold self.threshold = threshold # condition object, wait on this self.thresh_cond = mp.Condition() self.sentinel = sentinel
def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. :param callable func: object to call after acquiring the lock :param str lock_path: path to file or directory to lock """ # Reload module to reset internal _LOCKS dictionary reload_module(util) # start child and wait for it to grab the lock cv = multiprocessing.Condition() cv.acquire() child_args = ( cv, lock_path, ) child = multiprocessing.Process(target=hold_lock, args=child_args) child.start() cv.wait() # call func and terminate the child func() cv.notify() cv.release() child.join() assert child.exitcode == 0
def __init__(self, manager, name=None, maxsize=0, timeout=None): # https://stackoverflow.com/questions/39496554/cannot-subclass-multiprocessing-queue-in-python-3-5/ if sys.version_info.major == 2: super(ProcessChannel, self).__init__(maxsize=maxsize) elif sys.version_info.major == 3: super(ProcessChannel, self).__init__( maxsize=maxsize, ctx=multiprocessing.get_context()) else: raise Exception("Error Python version") self._maxsize = maxsize self._timeout = timeout self.name = name self._stop = False self._cv = multiprocessing.Condition() self._producers = [] self._pushed_producer_count = manager.dict() # {data_id: count} self._input_buf = manager.dict() # {data_id: {op_name: data}} self._reset_max_cursor = 1000000000000000000 self._consumer_cursors = manager.dict() # {op_name: cursor} self._cursor_count = manager.dict() # {cursor: count} self._base_cursor = manager.Value('i', 0) self._output_buf = manager.list()
def test_lock_blocking(): tempdir = tempfile.mkdtemp() try: cond = multiprocessing.Condition() def target_wlock(): setup_prefix() try: cond.acquire() trylock_uuid(tempdir, True) cond.notify() cond.release() time.sleep(0.5) finally: teardown_prefix() sys.exit(0) proc = multiprocessing.Process( target=target_wlock ) start = time.time() cond.acquire() proc.start() cond.wait() lock_uuid(tempdir, True) end = time.time() proc.join() tools.assert_true(end - start > 0.5) finally: shutil.rmtree(tempdir)
def __init__(self, manager, name=None, maxsize=0, channel_recv_frist_arrive=False): # For queue multiprocess: after putting an object on # an empty queue there may be an infinitessimal delay # before the queue's :meth:`~Queue.empty` # see more: # - https://bugs.python.org/issue18277 # - https://hg.python.org/cpython/rev/860fc6a2bd21 self._que = manager.PriorityQueue(maxsize=maxsize) self._maxsize = maxsize self.name = name self._stop = manager.Value('i', 0) self._cv = multiprocessing.Condition() self._producers = [] self._pushed_producer_count = manager.dict() # {data_id: count} self._input_buf = manager.dict() # {data_id: {op_name: data}} self._reset_max_cursor = 1000000000000000000 self._consumer_cursors = manager.dict() # {op_name: cursor} self._cursor_count = manager.dict() # {cursor: count} self._base_cursor = manager.Value('i', 0) self._output_buf = manager.list() self._cur_max_dataid = manager.Value('i', -1) self._channel_recv_frist_arrive = channel_recv_frist_arrive
def __init__(self, number_of_dequeuers): self._cond = mp.Condition() self._queue = mp.Queue() self._initial_producer_done = mp.Value(ctypes.c_bool, False) self._donations_possible = mp.Value(ctypes.c_bool, True) self._want_donations = mp.Value(ctypes.c_bool, False) self._number_busy = mp.Value(ctypes.c_uint, number_of_dequeuers)
def __init__(self, framestream): self.framestream = framestream self.objs = [] #self.i=None#iter(self.framestream) self.thread = None self.savedFrames = [] self.q = Queue.Queue(1) if self.multiprocessable: self.nextlock = multiprocessing.Condition() self.doSave = multiprocessing.Event() self.loadEvent = multiprocessing.Event() self.terminated = multiprocessing.Event() self.finished = multiprocessing.Event() self.threadstartlock = multiprocessing.Lock() self.threadstarted = multiprocessing.Event() self.queueclass = multiprocessing.JoinableQueue #LocalOrRemoteQueue else: self.nextlock = threading.Condition() self.doSave = threading.Event() self.loadEvent = threading.Event() self.terminated = threading.Event() self.finished = threading.Event() self.threadstartlock = threading.Lock() self.threadstarted = threading.Event() self.queueclass = Queue.Queue self.thread = None # iq.IterableToQueueThread(framestream,self.q,inSync=True)#self.terminator)#,#threading.Thread(target=brancher_nextloop,args=[weakref.ref(self)]) self.pid = None #self.startThread() self.provides = framestream.provides if self.provides == None: raise RuntimeError('No provides provided')
def test_non_atomically_pickleables_multiprocessing(): ''' Test `is_atomically_pickleable` on non-atomically pickleable objects. Not including `multiprocessing` objects. ''' if not import_tools.exists('multiprocessing'): raise nose.SkipTest('`multiprocessing` is not installed.') import multiprocessing non_pickleables = [ multiprocessing.Lock(), multiprocessing.BoundedSemaphore(), multiprocessing.Condition(), multiprocessing.JoinableQueue(), multiprocessing.Pool(), multiprocessing.Queue(), multiprocessing.RLock(), multiprocessing.Semaphore(), ] for thing in non_pickleables: assert not pickle_tools.is_atomically_pickleable(thing) assert not is_pickle_successful(thing) assert not pickle_tools.is_atomically_pickleable(NonPickleableObject())
def __init__(self): setprocname('Lights server') set_IRL_sunset() for room in ROOMS.values(): room.resetFromLoggedState() self.bulb_event = mp.Event() self.wake_condition = mp.Condition() self.TIMEOUT_INTERVAL = 5*60 # 5 min self.monitor_bulb_advert_proc = mp.Process(target=monitor_advert_bulbs, args=(self.bulb_event, self.wake_condition,)) self.monitor_bulb_static_proc = mp.Process(target=monitor_bulb_static, args=(self.bulb_event, self.wake_condition,)) self.ping_event = mp.Event() self.ping_pipe, ping_child_pipe = mp.Pipe() self.check_ping_proc = mp.Process(target=checkPingThreaded, args=(self.ping_event, ping_child_pipe, self.wake_condition,pcStatus, phoneStatus,)) self.ping_res = True self.timer_wake = False self.switch_event = mp.Event() self.switch_pipe, switch_child_pipe = mp.Pipe() self.monitor_switches_proc = mp.Process(target=monitor_switches, args=(self.switch_event, self.wake_condition, switch_child_pipe, )) self.switch_room = None self.switch_action = None self.http_event = mp.Event() self.http_pipe, http_child_pipe = mp.Pipe() self.http_proc = mp.Process(target=http_server, args=(self.http_event, self.wake_condition, http_child_pipe, )) self.http_res = None signal.signal(signal.SIGTERM, self.graceful_shutdown)
def __init__(self, calibration_data=None, axis_remap=None, sensor_update_frequency_hz=20.0): # Initialize BNO sensor with up to one retry try: # Create and configure the BNO sensor connection. # Raspberry Pi configuration with serial UART and RST connected to GPIO 18: self._sensor = BNO055.BNO055(serial_port='/dev/ttyAMA0', rst=18) self._sensor.begin() except RuntimeError: # Second try, just in case self._sensor = BNO055.BNO055(serial_port='/dev/ttyAMA0', rst=18) self._sensor.begin() if axis_remap is not None: self._sensor.set_axis_remap(**axis_remap) if calibration_data is not None: self._sensor.set_calibration(calibration_data) # The 5 elements of the shared memory array are (update count, update time, linear_acceleration_x, linear_acceleration_y, linear_acceleration_z) self._multiproc_shared_data = multiproc.Array('d', 5) self._condition = multiproc.Condition() self._state = {} self.sensor_update_frequency_hz = sensor_update_frequency_hz # TODO: it's probably bad form to spawn a process within the constructor of an object. Could move this # to a public start() method sensor_proc = multiproc.Process( target=read_bno_multiproc, args=(self._sensor, self.sensor_update_frequency_hz, self._multiproc_shared_data, self._condition)) sensor_proc.daemon = True # Don't let the BNO reading thread block exiting. sensor_proc.start()
def __init__(self, spike_sender, place_field_provider): """TODO: to be defined1. """ ThreadExtension.StoppableProcess.__init__(self) # Hoping that everything in python is pass by reference. Place fields # is a giant array! Both spike buffer and place fields are shared # resources. self.prob_buffer_size = 100 self.time_bin_width = int( 0.01 * 30000) #trodes samples at 30kHz, so this is 10ms self.nspks_until_get_new_pf = 50 self.num_return_time_bins = 10 self._spike_buffer = spike_sender.get_spike_buffer_connection() #self._log_place_fields = place_field_provider.get_log_place_fields() self._place_field_provider = place_field_provider self._bin_times = np.zeros((1, self.prob_buffer_size)) self._done_exp_output_flag = [ True for i in range(self.prob_buffer_size) ] self._need_exp_output_flag = [ False for i in range(self.prob_buffer_size) ] self.time_bin = 0 self._log_probs_out = np.zeros( (self.prob_buffer_size, PositionAnalysis.N_POSITION_BINS[0], PositionAnalysis.N_POSITION_BINS[1])) self._probs_out = np.add( np.zeros_like(self._log_probs_out), 1.0 / (PositionAnalysis.N_POSITION_BINS[0] + PositionAnalysis.N_POSITION_BINS[1])) self._output_lock = multiprocessing.Condition()
def __init__( self, network_controller, cvkernel_json_path, cvsupervisor_json_path, # Определение run_state_handler, ready_state_handler, closed_state_handler): # конструктора класса self.cv_supervision_params = json.load( open(cvsupervisor_json_path, 'r')) # Загрузка параметров супервидения self.display_processes = { } # Создание словаря процессов отображения оверлея self.supervision_tcp = cv_network_controller.create_tcp_sock( int(self.cv_supervision_params['port']) ) # Создание TCP сервера принятия супервизионных данных от ядра self.supervision_mutex = multiprocessing.RLock( ) # Определение мьютекса процесса супервидения self.supervision_cv = multiprocessing.Condition( self.supervision_mutex ) # Определение условной блокировки процесса супервидения self.supervision = multiprocessing.Process( target=self.__supervision_process ) # Определение процесса супервидения self.supervision.start() # Запуск процесса cv_connector.__init__( # Вызов self, network_controller, cvkernel_json_path, run_state_handler, # конструктора ready_state_handler, closed_state_handler, {'type': 'supervisor'}, # базового self.cv_supervision_params # класса )
def generator(sample_ids, data_locations, data_reader_info, preprocessor_info, sample_weighter_info, **kwargs): batch_queue = multiprocessing.Queue(maxsize=kwargs["mp_info"]["max_queue_size"]) end_epoch_condition = multiprocessing.Condition() data_producer_kwargs = deepcopy(kwargs) data_producer_kwargs["end_epoch_condition"] = end_epoch_condition workers = [] for worker_id in range(kwargs["mp_info"]["num_of_workers"]): workers.append(GenericDataProducer(ExampleBatchManager(batch_queue, sample_ids, data_locations, kwargs["chunk_info"]["chunks"][worker_id], kwargs["batch_size"]), data_reader_info, preprocessor_info=preprocessor_info, sample_weighter_info=sample_weighter_info, **data_producer_kwargs)) workers[-1].start() samples_read_in_epoch = 0 completed_epoch = kwargs["epoch_info"]["last_completed_epoch"] while True: current_batch = batch_queue.get() samples_read_in_epoch += len(current_batch[0]) print("\n" + str(samples_read_in_epoch) + "\n") if samples_read_in_epoch == kwargs["chunk_info"]["samples_per_epoch"]: completed_epoch += 1 samples_read_in_epoch=0 if completed_epoch == kwargs["epoch_info"]["target_epoch"]: for worker in workers: worker.terminate() batch_queue.close() yield current_batch break else: handle_epoch_end(sample_ids, end_epoch_condition, kwargs["epoch_info"]["epoch_end_handler_info"]) yield current_batch else: yield current_batch
def __init__(self, name, password='', password2=None, key=None): """ Constructor. The authorization password is only required on hosts that separate the authentication from the authorization procedure. If an authorization password is not given, it defaults to the same value as the authentication password. :type name: string :param name: A username. :type password: string :param password: The authentication password. :type password2: string :param password2: The authorization password, if required. :type key: PrivateKey :param key: A private key, if required. """ self.acquired_event = Event() self.released_event = Event() self.changed_event = Event() self.name = name self.password = password self.authorization_password = password2 self.key = key self.synclock = multiprocessing.Condition(multiprocessing.Lock()) self.lock = multiprocessing.Lock()
def test_pipespeed(): c, d = multiprocessing.Pipe() cond = multiprocessing.Condition() elapsed = 0 iterations = 1 while elapsed < delta: iterations *= 2 p = multiprocessing.Process(target=pipe_func, args=(d, cond, iterations)) cond.acquire() p.start() cond.wait() cond.release() result = None t = _timer() while result != 'STOP': result = c.recv() elapsed = _timer() - t p.join() print iterations, 'objects passed through connection in', elapsed, 'seconds' print 'average number/sec:', iterations / elapsed
def setup_heartbeat(self, client_controller): cond = multiprocessing.Condition() s_init_finish = Value('i', 0) do_sample = Value('i', 0) do_sample_lock = Lock() server_process = multiprocessing.Process(target=self.server_heart_beat, args=(cond, s_init_finish, do_sample, do_sample_lock)) server_process.daemon = False server_process.start() logger.info("Waiting for server init ...") cond.acquire() while (s_init_finish.value == 0): cond.wait() if s_init_finish.value == 5: logger.error("Waiting for server init ... FAIL") raise RuntimeError("server init failed.") cond.release() logger.info("Waiting for server init ... Done") # let all clients start running the benchmark client_controller.client_run(do_sample, do_sample_lock) cond.acquire() s_init_finish.value = 0 cond.release() return server_process
def init(self, size, name=None, lock=None, not_full=None, not_empty=None): self.size = size if self.size & (self.size - 1): self.size = roundup_pow_of_two(self.size) self.lock = lock or mp.Lock() self.not_full = not_full or mp.Condition(self.lock) self.not_empty = not_empty or mp.Condition(self.lock) ss = SharedStructure() ss.add("in_", 1, c_uint32) ss.add("out_", 1, c_uint32) ss.add("buf", self.size, c_uint8) sm, d = ss.alloc(name) self.sm = sm self.__in = d.in_ self.__out = d.out_ self._buffer = d.buf
def __init__(self): self.cond = multiprocessing.Condition() self.p = multiprocessing.Process(target=self.worker) self.q = multiprocessing.Queue(maxsize=20) self.total_job = 0 self.p.start()