def _init_workers(self):
jt.clean()
jt.gc()
self.index_list = mp.Array('i', self.real_len, lock=False)
workers = []
# batch id to worker id
self.idmap = mp.Array('i', self.batch_len, lock=False)
# global token index
self.gid = mp.Value('i', self.batch_len)
# global token index condition
self.gidc = mp.Condition(self.gid.get_lock())
# number of idle workers
self.num_idle = mp.Value('i', 0, lock=False)
# number of idle workers condition
self.num_idle_c = mp.Condition(self.gid.get_lock())
for i in range(self.num_workers):
w = Worker(target=self._worker_main,
args=(i, ),
buffer_size=self.buffer_size,
keep_numpy_array=self.keep_numpy_array)
workers.append(w)
self.workers = workers
self.index_list_numpy = np.ndarray(dtype='int32',
shape=self.real_len,
buffer=self.index_list)
def test_check_for_downloads_from_md5():
lpath = 'lpath'
rfile = azmodels.StorageEntity('cont')
rfile._md5 = 'abc'
rfile._client = mock.MagicMock()
rfile._client.primary_endpoint = 'ep'
rfile._name = 'name'
rfile._vio = None
rfile._size = 256
key = ops.Downloader.create_unique_transfer_operation_id(rfile)
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [
None,
(key, lpath, rfile._size, False),
]
d._add_to_download_queue = mock.MagicMock()
d._all_remote_files_processed = False
d._download_terminate = True
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 0
with mock.patch(
'blobxfer.operations.download.Downloader.'
'termination_check_md5',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [
None,
(key, lpath, rfile._size, False),
]
d._add_to_download_queue = mock.MagicMock()
patched_tc.side_effect = [False, False, True]
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 1
with mock.patch(
'blobxfer.operations.download.Downloader.'
'termination_check_md5',
new_callable=mock.PropertyMock) as patched_tc:
d = ops.Downloader(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock())
d._md5_map[key] = rfile
d._transfer_set.add(key)
d._md5_offload = mock.MagicMock()
d._md5_offload.done_cv = multiprocessing.Condition()
d._md5_offload.pop_done_queue.side_effect = [None]
d._add_to_download_queue = mock.MagicMock()
patched_tc.side_effect = [False, True, True]
d._check_for_downloads_from_md5()
assert d._add_to_download_queue.call_count == 0
def __init__(self):
self.ids = multiprocessing.Array('i', 10, lock=False)
self.digs = multiprocessing.Array('i', 10, lock=False)
self.active = multiprocessing.Array('i', 10, lock=False)
self.total_digs = multiprocessing.Value('i', lock=False)
self.total_active = multiprocessing.Value('i', lock=False)
self.lock = multiprocessing.Lock()
self.producer_cond = multiprocessing.Condition(self.lock)
self.consumer_cond = multiprocessing.Condition(self.lock)
def __init__(self, maxsize=0):
'''initialize the queue'''
self.mutex = multiprocessing.Lock()
self.not_empty = multiprocessing.Condition(self.mutex)
self.not_full = multiprocessing.Condition(self.mutex)
self.maxsize = maxsize
self._tags = {} # list of refid's for each tag
self._queue = {} # the actual queue data
self._refcount = {
} # how many tags refer to a given refid in the queue
self.id_generator = id_generator()
def __init__(self, max_cpu):
self.__max_cpu = max_cpu
self.__runing = multiprocessing.Value('I', 0, lock=True)
self.__close = multiprocessing.Value('I', 0, lock=True)
# 进程计数
self.__count = multiprocessing.Value('I', 0, lock=True)
# 结束条件变量
self.__finish_cond = multiprocessing.Condition()
# 单个进程结束条件变量
self.__count_cond = multiprocessing.Condition()
self.__task_que = multiprocessing.Queue()
def test():
manager = multiprocessing.Manager()
gc.disable()
print '\n\t######## testing Queue.Queue\n'
test_queuespeed(threading.Thread, Queue.Queue(),
threading.Condition())
print '\n\t######## testing multiprocessing.Queue\n'
test_queuespeed(multiprocessing.Process, multiprocessing.Queue(),
multiprocessing.Condition())
print '\n\t######## testing Queue managed by server process\n'
test_queuespeed(multiprocessing.Process, manager.Queue(),
manager.Condition())
print '\n\t######## testing multiprocessing.Pipe\n'
test_pipespeed()
print
print '\n\t######## testing list\n'
test_seqspeed(range(10))
print '\n\t######## testing list managed by server process\n'
test_seqspeed(manager.list(range(10)))
print '\n\t######## testing Array("i", ..., lock=False)\n'
test_seqspeed(multiprocessing.Array('i', range(10), lock=False))
print '\n\t######## testing Array("i", ..., lock=True)\n'
test_seqspeed(multiprocessing.Array('i', range(10), lock=True))
print
print '\n\t######## testing threading.Lock\n'
test_lockspeed(threading.Lock())
print '\n\t######## testing threading.RLock\n'
test_lockspeed(threading.RLock())
print '\n\t######## testing multiprocessing.Lock\n'
test_lockspeed(multiprocessing.Lock())
print '\n\t######## testing multiprocessing.RLock\n'
test_lockspeed(multiprocessing.RLock())
print '\n\t######## testing lock managed by server process\n'
test_lockspeed(manager.Lock())
print '\n\t######## testing rlock managed by server process\n'
test_lockspeed(manager.RLock())
print
print '\n\t######## testing threading.Condition\n'
test_conditionspeed(threading.Thread, threading.Condition())
print '\n\t######## testing multiprocessing.Condition\n'
test_conditionspeed(multiprocessing.Process, multiprocessing.Condition())
print '\n\t######## testing condition managed by a server process\n'
test_conditionspeed(multiprocessing.Process, manager.Condition())
gc.enable()
def __init__(self):
""" Initialize lock state
"""
# NOTE: If ever we want an equal access priority instead of giving
# access priority to the writers, we could have a global queue FIFO
# queue instead of just the pending writers queue. This would move this
# from a case-2 RWLock problem to a case-3, ensuring zero starvation of
# either lock.
# Condition lock on internal variables
# noinspection PyUnresolvedReferences
self.__cond = multiprocessing.Condition()
# Records the ID of the current writer. If this is None, there isn't
# currently and entity that has the write lock.
#: :type: None or int
self.__writer = None
# Allows write locks to be reentrant. The write lock is only released
# when this hits zero during a releaseWrite
self.__writer_count = 0
# Writers will be serviced before readers, recorded by this count.
# Writers serviced in order of request as ordered in this list.
#: :type: deque of (int, int)
self.__pending_writers = deque() # insert linked-list structure here?
# current processes holding reader locks. Maps the user's ID to the
# number of times it has reentered reading locks.
#: :type: dict of (int, int)
self.__readers = {}
def __init__(self, is_threaded=True):
if is_threaded:
self._read_ready = threading.Condition(threading.Lock())
else:
self._read_ready = multiprocessing.Condition(multiprocessing.Lock())
self._readers = 0
def _acquire_join_entry(self, i: int, j: int) -> None:
self._ensure_join_entry(i, j)
if self._join_matrix[i][j] is None:
self._join_matrix[i][j] = multiprocessing.Condition()
self._join_matrix[i][j].acquire()
else:
raise RuntimeError('PANIC!')
def test_worker():
top_k_ready = multiprocessing.Condition()
with multiprocessing.Manager() as m:
top_k = m.dict()
count_q = multiprocessing.Queue()
vocab_q = multiprocessing.Queue()
worker = bpe.Worker(top_k_ready, top_k, count_q, vocab_q, 'aaabb', 100,
2)
worker.start()
assert vocab_q.get() == {b'a', b'b'}
counts = count_q.get()
assert counts == {(b'a', b'a'): 2, (b'a', b'b'): 1, (b'b', b'b'): 1}
top_k.update(
{x[0]: b''.join(x[0])
for x in Counter(counts).most_common(2)})
with top_k_ready:
top_k_ready.notify()
# Round 2.
counts = count_q.get()
assert counts == {(b'aa', b'ab'): 1, (b'ab', b'b'): 1}
# Finish.
top_k.clear()
with top_k_ready:
top_k_ready.notify()
worker.join()
assert not worker.is_alive()
def __init__(self, size):
self.size = size
self.l = mp.Value(ctypes.c_longlong, 0, lock=False)
self.r = mp.Value(ctypes.c_longlong, 0, lock=False)
self.is_full = mp.Value(ctypes.c_bool, False, lock=False)
self.lock = mp.Lock()
self.cv = mp.Condition(self.lock)
def init_workers(self):
"""
Initialize all types of workers and start their worker processes.
"""
actor_queues = [MpQueue(2 * 1000 * 1000) for _ in range(self.cfg.num_workers)]
policy_worker_queues = dict()
for policy_id in range(self.cfg.num_policies):
policy_worker_queues[policy_id] = []
for i in range(self.cfg.policy_workers_per_policy):
policy_worker_queues[policy_id].append(TorchJoinableQueue())
log.info('Initializing learners...')
policy_locks = [multiprocessing.Lock() for _ in range(self.cfg.num_policies)]
resume_experience_collection_cv = [multiprocessing.Condition() for _ in range(self.cfg.num_policies)]
learner_idx = 0
for policy_id in range(self.cfg.num_policies):
learner_worker = LearnerWorker(
learner_idx, policy_id, self.cfg, self.obs_space, self.action_space,
self.report_queue, policy_worker_queues[policy_id], self.traj_buffers,
policy_locks[policy_id], resume_experience_collection_cv[policy_id],
)
learner_worker.start_process()
learner_worker.init()
self.learner_workers[policy_id] = learner_worker
learner_idx += 1
log.info('Initializing policy workers...')
for policy_id in range(self.cfg.num_policies):
self.policy_workers[policy_id] = []
policy_queue = MpQueue()
self.policy_queues[policy_id] = policy_queue
for i in range(self.cfg.policy_workers_per_policy):
policy_worker = PolicyWorker(
i, policy_id, self.cfg, self.obs_space, self.action_space, self.traj_buffers,
policy_queue, actor_queues, self.report_queue, policy_worker_queues[policy_id][i],
policy_locks[policy_id], resume_experience_collection_cv[policy_id],
)
self.policy_workers[policy_id].append(policy_worker)
policy_worker.start_process()
log.info('Initializing actors...')
# We support actor worker initialization in groups, which can be useful for some envs that
# e.g. crash when too many environments are being initialized in parallel.
# Currently the limit is not used since it is not required for any envs supported out of the box,
# so we parallelize initialization as hard as we can.
# If this is required for your environment, perhaps a better solution would be to use global locks,
# like FileLock (see doom_gym.py)
self.actor_workers = []
max_parallel_init = int(1e9) # might be useful to limit this for some envs
worker_indices = list(range(self.cfg.num_workers))
for i in range(0, self.cfg.num_workers, max_parallel_init):
workers = self.init_subset(worker_indices[i:i + max_parallel_init], actor_queues)
self.actor_workers.extend(workers)
def __init__(self, threshold=0, maxsize=0, sentinel=SENTINEL):
"""
A Thresholded Queue.
It provides the Condition object thesh_cond.
Threads waiting on thresh_cond are notified
whenever the number of elements (qsize)
falls below the specified threshold.
Input:
threshold threshold condition
maxsize maximum number of elements (put blocks if full)
"""
# Queue is an old style class, we can not use super
Queue.Queue.__init__(self, maxsize=maxsize)
logger.info('Init queue maxsize: %d, threshold: %d',
maxsize, threshold)
# the threshold
self.threshold = threshold
# condition object, wait on this
self.thresh_cond = mp.Condition()
self.sentinel = sentinel
def lock_and_call(func, lock_path):
"""Grab a lock for lock_path and call func.
:param callable func: object to call after acquiring the lock
:param str lock_path: path to file or directory to lock
"""
# Reload module to reset internal _LOCKS dictionary
reload_module(util)
# start child and wait for it to grab the lock
cv = multiprocessing.Condition()
cv.acquire()
child_args = (
cv,
lock_path,
)
child = multiprocessing.Process(target=hold_lock, args=child_args)
child.start()
cv.wait()
# call func and terminate the child
func()
cv.notify()
cv.release()
child.join()
assert child.exitcode == 0
def __init__(self, manager, name=None, maxsize=0, timeout=None):
# https://stackoverflow.com/questions/39496554/cannot-subclass-multiprocessing-queue-in-python-3-5/
if sys.version_info.major == 2:
super(ProcessChannel, self).__init__(maxsize=maxsize)
elif sys.version_info.major == 3:
super(ProcessChannel, self).__init__(
maxsize=maxsize, ctx=multiprocessing.get_context())
else:
raise Exception("Error Python version")
self._maxsize = maxsize
self._timeout = timeout
self.name = name
self._stop = False
self._cv = multiprocessing.Condition()
self._producers = []
self._pushed_producer_count = manager.dict() # {data_id: count}
self._input_buf = manager.dict() # {data_id: {op_name: data}}
self._reset_max_cursor = 1000000000000000000
self._consumer_cursors = manager.dict() # {op_name: cursor}
self._cursor_count = manager.dict() # {cursor: count}
self._base_cursor = manager.Value('i', 0)
self._output_buf = manager.list()
def test_lock_blocking():
tempdir = tempfile.mkdtemp()
try:
cond = multiprocessing.Condition()
def target_wlock():
setup_prefix()
try:
cond.acquire()
trylock_uuid(tempdir, True)
cond.notify()
cond.release()
time.sleep(0.5)
finally:
teardown_prefix()
sys.exit(0)
proc = multiprocessing.Process(
target=target_wlock
)
start = time.time()
cond.acquire()
proc.start()
cond.wait()
lock_uuid(tempdir, True)
end = time.time()
proc.join()
tools.assert_true(end - start > 0.5)
finally:
shutil.rmtree(tempdir)
def __init__(self,
manager,
name=None,
maxsize=0,
channel_recv_frist_arrive=False):
# For queue multiprocess: after putting an object on
# an empty queue there may be an infinitessimal delay
# before the queue's :meth:`~Queue.empty`
# see more:
# - https://bugs.python.org/issue18277
# - https://hg.python.org/cpython/rev/860fc6a2bd21
self._que = manager.PriorityQueue(maxsize=maxsize)
self._maxsize = maxsize
self.name = name
self._stop = manager.Value('i', 0)
self._cv = multiprocessing.Condition()
self._producers = []
self._pushed_producer_count = manager.dict() # {data_id: count}
self._input_buf = manager.dict() # {data_id: {op_name: data}}
self._reset_max_cursor = 1000000000000000000
self._consumer_cursors = manager.dict() # {op_name: cursor}
self._cursor_count = manager.dict() # {cursor: count}
self._base_cursor = manager.Value('i', 0)
self._output_buf = manager.list()
self._cur_max_dataid = manager.Value('i', -1)
self._channel_recv_frist_arrive = channel_recv_frist_arrive
def __init__(self, number_of_dequeuers):
self._cond = mp.Condition()
self._queue = mp.Queue()
self._initial_producer_done = mp.Value(ctypes.c_bool, False)
self._donations_possible = mp.Value(ctypes.c_bool, True)
self._want_donations = mp.Value(ctypes.c_bool, False)
self._number_busy = mp.Value(ctypes.c_uint, number_of_dequeuers)
def __init__(self, framestream):
self.framestream = framestream
self.objs = []
#self.i=None#iter(self.framestream)
self.thread = None
self.savedFrames = []
self.q = Queue.Queue(1)
if self.multiprocessable:
self.nextlock = multiprocessing.Condition()
self.doSave = multiprocessing.Event()
self.loadEvent = multiprocessing.Event()
self.terminated = multiprocessing.Event()
self.finished = multiprocessing.Event()
self.threadstartlock = multiprocessing.Lock()
self.threadstarted = multiprocessing.Event()
self.queueclass = multiprocessing.JoinableQueue #LocalOrRemoteQueue
else:
self.nextlock = threading.Condition()
self.doSave = threading.Event()
self.loadEvent = threading.Event()
self.terminated = threading.Event()
self.finished = threading.Event()
self.threadstartlock = threading.Lock()
self.threadstarted = threading.Event()
self.queueclass = Queue.Queue
self.thread = None # iq.IterableToQueueThread(framestream,self.q,inSync=True)#self.terminator)#,#threading.Thread(target=brancher_nextloop,args=[weakref.ref(self)])
self.pid = None
#self.startThread()
self.provides = framestream.provides
if self.provides == None:
raise RuntimeError('No provides provided')
def test_non_atomically_pickleables_multiprocessing():
'''
Test `is_atomically_pickleable` on non-atomically pickleable objects.
Not including `multiprocessing` objects.
'''
if not import_tools.exists('multiprocessing'):
raise nose.SkipTest('`multiprocessing` is not installed.')
import multiprocessing
non_pickleables = [
multiprocessing.Lock(),
multiprocessing.BoundedSemaphore(),
multiprocessing.Condition(),
multiprocessing.JoinableQueue(),
multiprocessing.Pool(),
multiprocessing.Queue(),
multiprocessing.RLock(),
multiprocessing.Semaphore(),
]
for thing in non_pickleables:
assert not pickle_tools.is_atomically_pickleable(thing)
assert not is_pickle_successful(thing)
assert not pickle_tools.is_atomically_pickleable(NonPickleableObject())
def __init__(self):
setprocname('Lights server')
set_IRL_sunset()
for room in ROOMS.values():
room.resetFromLoggedState()
self.bulb_event = mp.Event()
self.wake_condition = mp.Condition()
self.TIMEOUT_INTERVAL = 5*60 # 5 min
self.monitor_bulb_advert_proc = mp.Process(target=monitor_advert_bulbs, args=(self.bulb_event, self.wake_condition,))
self.monitor_bulb_static_proc = mp.Process(target=monitor_bulb_static, args=(self.bulb_event, self.wake_condition,))
self.ping_event = mp.Event()
self.ping_pipe, ping_child_pipe = mp.Pipe()
self.check_ping_proc = mp.Process(target=checkPingThreaded, args=(self.ping_event, ping_child_pipe, self.wake_condition,pcStatus, phoneStatus,))
self.ping_res = True
self.timer_wake = False
self.switch_event = mp.Event()
self.switch_pipe, switch_child_pipe = mp.Pipe()
self.monitor_switches_proc = mp.Process(target=monitor_switches, args=(self.switch_event, self.wake_condition, switch_child_pipe, ))
self.switch_room = None
self.switch_action = None
self.http_event = mp.Event()
self.http_pipe, http_child_pipe = mp.Pipe()
self.http_proc = mp.Process(target=http_server, args=(self.http_event, self.wake_condition, http_child_pipe, ))
self.http_res = None
signal.signal(signal.SIGTERM, self.graceful_shutdown)
def __init__(self,
calibration_data=None,
axis_remap=None,
sensor_update_frequency_hz=20.0):
# Initialize BNO sensor with up to one retry
try:
# Create and configure the BNO sensor connection.
# Raspberry Pi configuration with serial UART and RST connected to GPIO 18:
self._sensor = BNO055.BNO055(serial_port='/dev/ttyAMA0', rst=18)
self._sensor.begin()
except RuntimeError:
# Second try, just in case
self._sensor = BNO055.BNO055(serial_port='/dev/ttyAMA0', rst=18)
self._sensor.begin()
if axis_remap is not None:
self._sensor.set_axis_remap(**axis_remap)
if calibration_data is not None:
self._sensor.set_calibration(calibration_data)
# The 5 elements of the shared memory array are (update count, update time, linear_acceleration_x, linear_acceleration_y, linear_acceleration_z)
self._multiproc_shared_data = multiproc.Array('d', 5)
self._condition = multiproc.Condition()
self._state = {}
self.sensor_update_frequency_hz = sensor_update_frequency_hz
# TODO: it's probably bad form to spawn a process within the constructor of an object. Could move this
# to a public start() method
sensor_proc = multiproc.Process(
target=read_bno_multiproc,
args=(self._sensor, self.sensor_update_frequency_hz,
self._multiproc_shared_data, self._condition))
sensor_proc.daemon = True # Don't let the BNO reading thread block exiting.
sensor_proc.start()
def __init__(self, spike_sender, place_field_provider):
"""TODO: to be defined1. """
ThreadExtension.StoppableProcess.__init__(self)
# Hoping that everything in python is pass by reference. Place fields
# is a giant array! Both spike buffer and place fields are shared
# resources.
self.prob_buffer_size = 100
self.time_bin_width = int(
0.01 * 30000) #trodes samples at 30kHz, so this is 10ms
self.nspks_until_get_new_pf = 50
self.num_return_time_bins = 10
self._spike_buffer = spike_sender.get_spike_buffer_connection()
#self._log_place_fields = place_field_provider.get_log_place_fields()
self._place_field_provider = place_field_provider
self._bin_times = np.zeros((1, self.prob_buffer_size))
self._done_exp_output_flag = [
True for i in range(self.prob_buffer_size)
]
self._need_exp_output_flag = [
False for i in range(self.prob_buffer_size)
]
self.time_bin = 0
self._log_probs_out = np.zeros(
(self.prob_buffer_size, PositionAnalysis.N_POSITION_BINS[0],
PositionAnalysis.N_POSITION_BINS[1]))
self._probs_out = np.add(
np.zeros_like(self._log_probs_out),
1.0 / (PositionAnalysis.N_POSITION_BINS[0] +
PositionAnalysis.N_POSITION_BINS[1]))
self._output_lock = multiprocessing.Condition()
def __init__(
self,
network_controller,
cvkernel_json_path,
cvsupervisor_json_path, # Определение
run_state_handler,
ready_state_handler,
closed_state_handler): # конструктора класса
self.cv_supervision_params = json.load(
open(cvsupervisor_json_path,
'r')) # Загрузка параметров супервидения
self.display_processes = {
} # Создание словаря процессов отображения оверлея
self.supervision_tcp = cv_network_controller.create_tcp_sock(
int(self.cv_supervision_params['port'])
) # Создание TCP сервера принятия супервизионных данных от ядра
self.supervision_mutex = multiprocessing.RLock(
) # Определение мьютекса процесса супервидения
self.supervision_cv = multiprocessing.Condition(
self.supervision_mutex
) # Определение условной блокировки процесса супервидения
self.supervision = multiprocessing.Process(
target=self.__supervision_process
) # Определение процесса супервидения
self.supervision.start() # Запуск процесса
cv_connector.__init__( # Вызов
self,
network_controller,
cvkernel_json_path,
run_state_handler, # конструктора
ready_state_handler,
closed_state_handler,
{'type': 'supervisor'}, # базового
self.cv_supervision_params # класса
)
def generator(sample_ids, data_locations, data_reader_info, preprocessor_info, sample_weighter_info, **kwargs):
batch_queue = multiprocessing.Queue(maxsize=kwargs["mp_info"]["max_queue_size"])
end_epoch_condition = multiprocessing.Condition()
data_producer_kwargs = deepcopy(kwargs)
data_producer_kwargs["end_epoch_condition"] = end_epoch_condition
workers = []
for worker_id in range(kwargs["mp_info"]["num_of_workers"]):
workers.append(GenericDataProducer(ExampleBatchManager(batch_queue, sample_ids, data_locations, kwargs["chunk_info"]["chunks"][worker_id], kwargs["batch_size"]), data_reader_info, preprocessor_info=preprocessor_info, sample_weighter_info=sample_weighter_info, **data_producer_kwargs))
workers[-1].start()
samples_read_in_epoch = 0
completed_epoch = kwargs["epoch_info"]["last_completed_epoch"]
while True:
current_batch = batch_queue.get()
samples_read_in_epoch += len(current_batch[0])
print("\n" + str(samples_read_in_epoch) + "\n")
if samples_read_in_epoch == kwargs["chunk_info"]["samples_per_epoch"]:
completed_epoch += 1
samples_read_in_epoch=0
if completed_epoch == kwargs["epoch_info"]["target_epoch"]:
for worker in workers:
worker.terminate()
batch_queue.close()
yield current_batch
break
else:
handle_epoch_end(sample_ids, end_epoch_condition, kwargs["epoch_info"]["epoch_end_handler_info"])
yield current_batch
else:
yield current_batch
def __init__(self, name, password='', password2=None, key=None):
"""
Constructor.
The authorization password is only required on hosts that
separate the authentication from the authorization procedure.
If an authorization password is not given, it defaults to the
same value as the authentication password.
:type name: string
:param name: A username.
:type password: string
:param password: The authentication password.
:type password2: string
:param password2: The authorization password, if required.
:type key: PrivateKey
:param key: A private key, if required.
"""
self.acquired_event = Event()
self.released_event = Event()
self.changed_event = Event()
self.name = name
self.password = password
self.authorization_password = password2
self.key = key
self.synclock = multiprocessing.Condition(multiprocessing.Lock())
self.lock = multiprocessing.Lock()
def test_pipespeed():
c, d = multiprocessing.Pipe()
cond = multiprocessing.Condition()
elapsed = 0
iterations = 1
while elapsed < delta:
iterations *= 2
p = multiprocessing.Process(target=pipe_func,
args=(d, cond, iterations))
cond.acquire()
p.start()
cond.wait()
cond.release()
result = None
t = _timer()
while result != 'STOP':
result = c.recv()
elapsed = _timer() - t
p.join()
print iterations, 'objects passed through connection in', elapsed, 'seconds'
print 'average number/sec:', iterations / elapsed
def setup_heartbeat(self, client_controller):
cond = multiprocessing.Condition()
s_init_finish = Value('i', 0)
do_sample = Value('i', 0)
do_sample_lock = Lock()
server_process = multiprocessing.Process(target=self.server_heart_beat,
args=(cond, s_init_finish,
do_sample,
do_sample_lock))
server_process.daemon = False
server_process.start()
logger.info("Waiting for server init ...")
cond.acquire()
while (s_init_finish.value == 0):
cond.wait()
if s_init_finish.value == 5:
logger.error("Waiting for server init ... FAIL")
raise RuntimeError("server init failed.")
cond.release()
logger.info("Waiting for server init ... Done")
# let all clients start running the benchmark
client_controller.client_run(do_sample, do_sample_lock)
cond.acquire()
s_init_finish.value = 0
cond.release()
return server_process
def init(self, size, name=None, lock=None, not_full=None, not_empty=None):
self.size = size
if self.size & (self.size - 1):
self.size = roundup_pow_of_two(self.size)
self.lock = lock or mp.Lock()
self.not_full = not_full or mp.Condition(self.lock)
self.not_empty = not_empty or mp.Condition(self.lock)
ss = SharedStructure()
ss.add("in_", 1, c_uint32)
ss.add("out_", 1, c_uint32)
ss.add("buf", self.size, c_uint8)
sm, d = ss.alloc(name)
self.sm = sm
self.__in = d.in_
self.__out = d.out_
self._buffer = d.buf
def __init__(self):
self.cond = multiprocessing.Condition()
self.p = multiprocessing.Process(target=self.worker)
self.q = multiprocessing.Queue(maxsize=20)
self.total_job = 0
self.p.start()
