def __init__(self, _registry: EventsProcessesRegistry):
self._registry = _registry
self._events_counter = multiprocessing.Value(ctypes.c_uint32, 0)
self._running = multiprocessing.Event()
self._sub_port = multiprocessing.Value(ctypes.c_uint16, 0)
self._queue = multiprocessing.Queue()
self._raw_events_lock = multiprocessing.RLock()
self.events_log_base_dir.mkdir(parents=True, exist_ok=True)
super().__init__(daemon=True)
def test():
manager = multiprocessing.Manager()
gc.disable()
print('\n\t######## testing Queue.Queue\n')
test_queuespeed(threading.Thread, queue.Queue(),
threading.Condition())
print('\n\t######## testing multiprocessing.Queue\n')
test_queuespeed(multiprocessing.Process, multiprocessing.Queue(),
multiprocessing.Condition())
print('\n\t######## testing Queue managed by server process\n')
test_queuespeed(multiprocessing.Process, manager.Queue(),
manager.Condition())
print('\n\t######## testing multiprocessing.Pipe\n')
test_pipespeed()
print()
print('\n\t######## testing list\n')
test_seqspeed(list(range(10)))
print('\n\t######## testing list managed by server process\n')
test_seqspeed(manager.list(list(range(10))))
print('\n\t######## testing Array("i", ..., lock=False)\n')
test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=False))
print('\n\t######## testing Array("i", ..., lock=True)\n')
test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=True))
print()
print('\n\t######## testing threading.Lock\n')
test_lockspeed(threading.Lock())
print('\n\t######## testing threading.RLock\n')
test_lockspeed(threading.RLock())
print('\n\t######## testing multiprocessing.Lock\n')
test_lockspeed(multiprocessing.Lock())
print('\n\t######## testing multiprocessing.RLock\n')
test_lockspeed(multiprocessing.RLock())
print('\n\t######## testing lock managed by server process\n')
test_lockspeed(manager.Lock())
print('\n\t######## testing rlock managed by server process\n')
test_lockspeed(manager.RLock())
print()
print('\n\t######## testing threading.Condition\n')
test_conditionspeed(threading.Thread, threading.Condition())
print('\n\t######## testing multiprocessing.Condition\n')
test_conditionspeed(multiprocessing.Process, multiprocessing.Condition())
print('\n\t######## testing condition managed by a server process\n')
test_conditionspeed(multiprocessing.Process, manager.Condition())
gc.enable()
def __init__(self):
self._lock = multiprocessing.RLock()
# Mapping of a key to an object
#: :type: dict of (str, object)
self._obj_cache = {}
# Mapping of a key to the last access time for that object (UNIX time)
#: :type: dict of (str, float)
self._obj_last_access = {}
# Mapping of timeout values for each entry
self._obj_timeouts = {}
def __init__(self, pid):
self.__watchermsg = multiprocessing.Manager().dict()
self.__WatchPid = pid
self.__watchermsg['MaxRam'] = 0
self.__watchermsg['Stop'] = False
self.__watchermsg['Begintime'] = 0
self.__watchermsg['Endtime'] = 0
self.__watchermsg['infertime'] = 0
self.__Lock = multiprocessing.RLock()
self.__WatchPeocess = multiprocessing.Process(name="Watcher",
target=self.WatchRam,
args=())
def __init__(self, connection, path):
#zk connection
self.connection = connection
self.path = path
self.deletion_handlers = {}
# Ensures path exists
self.connection.create_recursive(self.path + PREFIX,
"",
acl=zc.zk.OPEN_ACL_UNSAFE)
self.lock = multiprocessing.RLock()
# Stores leader queues
self.candidate_by_predecessor = {}
self.counter_by_candidate = {}
class AsyncFunctionWrapper(object):
_thread_pools = {}
_pool_lock = multiprocessing.RLock()
_async_pool_size = 10
def __init__(self):
self.max_tries = 1
#TODO: evaluate if this could be a classmethod;
def get_async(self, func, *args, **kwargs):
return self._get_wrapped_async_func(func, *args, **kwargs)
@classmethod
def _get_thread_pool(cls):
'''Get the thread pool for this process.'''
with cls._pool_lock:
try:
return cls._thread_pools[os.getpid()]
except KeyError:
pool = concurrent.futures.ThreadPoolExecutor(
cls._async_pool_size)
cls._thread_pools[os.getpid()] = pool
return pool
def _get_wrapped_async_func(self, func, *args, **kwargs):
'''Returns an asynchronous function wrapped around the given func.
The asynchronous call has a callback keyword added to it
'''
def AsyncWrapper(*args, **kwargs):
# Find the callback argument
try:
callback = kwargs['callback']
del kwargs['callback']
except KeyError:
if len(args) > 0 and hasattr(args[-1], '__call__'):
callback = args[-1]
args = args[:-1]
else:
raise AttributeError('A callback is necessary')
io_loop = tornado.ioloop.IOLoop.current()
def _cb(future, cur_try=0):
if future.exception() is None:
callback(future.result())
else:
_log.error('Error executing the function: %s' % future.exception())
raise future.exception()
future = AsyncFunctionWrapper._get_thread_pool().submit(
func, *args, **kwargs)
io_loop.add_future(future, _cb)
return AsyncWrapper
def __init__(self, data_file, out_file):
# Process the arguments to __init__.
self.data_file = data_file
self.out_file = out_file
tmp_home = os.path.expanduser('~/.sim_tmp')
if not os.path.exists(tmp_home):
os.mkdir(tmp_home)
self.tmp_dir = tempfile.mkdtemp(dir=tmp_home)
# Load and process the parameter on-disk.
mat = scipy.io.loadmat(data_file)
self.tr_file = str(mat['tr_file'][0])
self.te_file = str(mat['te_file'][0])
self.n_train = int(np.ravel(mat['nTrain']))
self.classes = np.ravel(mat['classes'])
self.eval_dir = str(mat['eval_dir'][0])
del mat, data_file, out_file
# Load the data matrices, the actual feature values and labels.
self.X_te = utils.make_mmap(self.tmp_dir, 'X_te.mmap', self.te_file, 'x')
gc.collect()
self.y_te = utils.make_mmap(self.tmp_dir, 'y_te.mmap', self.te_file, 'y')
gc.collect()
self.X_tr = utils.make_mmap(self.tmp_dir, 'X_tr.mmap', self.tr_file, 'x')
gc.collect()
self.y_tr = utils.make_mmap(self.tmp_dir, 'y_tr.mmap', self.tr_file, 'y')
gc.collect()
# Parallelize over up to 32 CPUs.
self.nCPUs = min(32, mp.cpu_count())
# Create queues to hold intermediate outputs.
self.q0 = mp.Queue()
self.q1 = mp.Queue()
self.q2 = mp.Queue()
# Create a lock to prevent races/deadlocks when printing to screen.
self.print_lock = mp.RLock()
gc.collect()
# create a sub-process for each part of the evaluation.
self.ps = [mp.Process(target=self.add_specs, args=())] + \
[mp.Process(target=self.specs_to_inputs, args=()) for _ in range(self.nCPUs)] + \
[mp.Process(target=self.run_wrapper, args=()) for _ in range(self.nCPUs)] + \
[mp.Process(target=self.save_data, args=())]
# tell the log that you've finished
print('initialized at', time.time())
def __init__(self, route_store):
super(DriverManager, self).__init__()
self._navigator = Navigator(route_store)
self._navigation_queue = deque(maxlen=10)
self._principal_steer_scale = 0
# Static conversion factor of m/s to km/h.
self._speed_scale = 3.6
self._cruise_speed_step = 0
self._steering_stabilizer = None
self._pilot_state = PilotState()
self._driver_cache = {}
self._lock = multiprocessing.RLock()
self._driver = None
self._driver_ctl = None
def __init__(self, accounts=None):
"""
Constructor.
:type accounts: Account|list[Account]
:param accounts: Passed to add_account()
"""
self.accounts = set()
self.unlocked_accounts = deque()
self.owner2account = defaultdict(list)
self.account2owner = dict()
self.unlock_cond = multiprocessing.Condition(multiprocessing.RLock())
if accounts:
self.add_account(accounts)
def __init__(self, port=9200, host="localhost", version=None) -> None:
super().__init__()
self._port = port
self._host = host
self._version = version or constants.ELASTICSEARCH_DEFAULT_VERSION
self._cluster = None
self._backend_port = None
self._proxy_thread = None
self._lifecycle_lock = multiprocessing.RLock()
self._started = False
self._stopped = multiprocessing.Event()
self._starting = multiprocessing.Event()
def setUp(self):
tmpFile = tempfile.NamedTemporaryFile(delete=False)
tmpFile.close()
self.dummyFileName = tmpFile.name
self.processList = []
try:
self.multiprocessing_lock = multiprocessing.RLock()
except ImportError:
self.multiprocessing_lock = None # *bsd platforms without proper synchronization primitives
self.multithreading_lock = threading.RLock()
def __init__(self, manager, elem_entries, size=5000):
from trainer_tool.buffers.buffer_utils import make_buffer
self.buffers, self.types, self.ctypes, self.shapes = make_buffer(
manager, elem_entries, size=size)
self.ptr = manager.mp_manager.Value('i', 0)
self.curr_size = manager.mp_manager.Value('i', 0)
self.max_size = size
# Use to coordinate processes in reading and writing and assures maximum 1 process reading at all times
self.write_lock = mp.RLock()
self.views = None
self.elem_entries = elem_entries
def install_s3mi(installed={}, mutex=TraceLock("install_s3mi", multiprocessing.RLock())): # pylint: disable=dangerous-default-value
with mutex:
if installed: # Mutable default value persists
return
try:
# This is typically a no-op.
command.execute(
"which s3mi || pip install git+git://github.com/chanzuckerberg/s3mi.git"
)
command.execute(
"s3mi tweak-vm || echo s3mi tweak-vm sometimes fails under docker. Continuing..."
)
finally:
installed['time'] = time.time()
def publish():
"""
The main render script.
"""
num_w = 56
black_list = Blacklist()
valid_data = construct_data_dirs(black_list)
print(valid_data)
print("Publishing segments: ")
num_segments = []
if E('errors.txt'):
os.remove('errors.txt')
try:
multiprocessing.freeze_support()
with multiprocessing.Pool(
num_w,
initializer=tqdm.tqdm.set_lock,
initargs=(multiprocessing.RLock(), )) as pool:
manager = ThreadManager(multiprocessing.Manager(), num_w, 1, 1)
func = functools.partial(_render_data, DATA_DIR, manager)
num_segments = list(
tqdm.tqdm(pool.imap_unordered(func, valid_data),
total=len(valid_data),
desc='Files',
miniters=1,
position=0,
maxinterval=1))
# for recording_name, render_path in tqdm.tqdm(valid_renders, desc='Files'):
# num_segments_rendered += gen_sarsa_pairs(render_path, recording_name, DATA_DIR)
except Exception as e:
if isinstance(e, KeyboardInterrupt):
pool.terminate()
pool.join()
raise e
print('\n' * num_w)
print("Exception in pool: ", type(e), e)
print('Vectorized {} files in total!'.format(sum(num_segments)))
raise e
num_segments_rendered = sum(num_segments)
print('\n' * num_w)
print('Vectorized {} files in total!'.format(num_segments_rendered))
if E('errors.txt'):
print('Errors:')
print(open('errors.txt', 'r').read())
def __init__(self):
if not '_ready' in dir(self):
# Der Konstruktor wird bei jeder Instanziierung aufgerufen.
# Einmalige Dinge wie zum Beispiel die Initialisierung von Klassenvariablen müssen also in diesen Block.
self._ready = True
self.lock = multiprocessing.RLock()
self.apps = {}
self.waitingCondition = multiprocessing.Condition()
self.manager = multiprocessing.Manager()
self.namespace = self.manager.Namespace()
self.appDeleteListener = AppDeleteListener(self,
self.waitingCondition,
self.namespace)
self.appDeleteListener.start()
def init_logger(logger, level=LOGLEVEL):
logger.setLevel(LOGLEVELS[level])
if not len(logger.handlers):
handler = logging.handlers.RotatingFileHandler(LOGFILE,
maxBytes=LOGFILESIZE,
backupCount=LOGFILES)
# This formatter removes the fractions of a second in the time.
# formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
# '%Y-%m-%d %H:%M:%S %Z')
# The default formatter contains fractions of a second in the time.
formatter = logging.Formatter(
'%(asctime)-15s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
handler.lock = multiprocessing.RLock()
logger.addHandler(handler)
def refreshed_credentials(credentials_mutex=multiprocessing.RLock(),
credentials_cache={}): # pylint: disable=dangerous-default-value
with credentials_mutex:
if credentials_cache.get("expiration_time", 0) < time.time() + 5 * 60:
try:
credentials_cache["vars"] = _get_credentials()
except:
log.write(
"ERROR: Failed to refresh credentials with boto, even after retries. Subcommands will have to do it themselves."
)
log.write(traceback.format_exc())
return {}
credentials_cache["expiration_time"] = time.time(
) + 15 * 60 # this is the default for most accounts
return credentials_cache["vars"]
def __init__(self):
self._lock = multiprocessing.RLock()
self._vars = {}
# Find the root directory of the source tree
cur_dir = os.path.abspath(os.path.dirname(__file__))
while cur_dir <> '/':
if os.path.exists(os.path.join(cur_dir, 'NEON_ROOT')):
self._NEON_ROOT = cur_dir
break
cur_dir = os.path.abspath(os.path.join(cur_dir, '..'))
if cur_dir == '/':
raise Error(
'Could not find the NEON_ROOT file in the source tree.')
def _initialize_context_lock(self, safety):
"""initialize context lock used for thread or process safety"""
if safety == 'thread':
self.lock = threading.RLock()
elif safety == 'process':
self.lock = multiprocessing.RLock()
elif safety is None:
# nullcontext in python3.7 and above
# see https://stackoverflow.com/a/45187287
if hasattr(contextlib, 'nullcontext'):
self.lock = contextlib.nullcontext()
else:
self.lock = contextlib.suppress()
else:
raise Exception('safety must be \'thread\', \'process\', or None')
def test_semaphore():
sema = multiprocessing.Semaphore(3)
mutex = multiprocessing.RLock()
running = multiprocessing.Value('i', 0)
processes = [
multiprocessing.Process(target=semaphore_func,
args=(sema, mutex, running)) for i in range(10)
]
for p in processes:
p.start()
for p in processes:
p.join()
def main():
"""
The main render script.
"""
# 1. Load the blacklist.
blacklist = set(np.loadtxt(BLACKLIST_PATH, dtype=np.str).tolist())
print("Constructing render directories.")
renders = construct_render_dirs(blacklist)
print("Validating metadata from files:")
valid_renders, invalid_renders = render_metadata(renders)
print(len(valid_renders))
# print("Rendering actions: ")
# valid_renders, invalid_renders = render_actions(valid_renders)
print("... found {} valid recordings and {} invalid recordings"
" out of {} total files".format(len(valid_renders),
len(invalid_renders),
len(os.listdir(MERGED_DIR))))
unfinished_renders = [
v for v in valid_renders if not len(glob.glob(J(v[1], '*.mp4')))
]
print("... found {} unfinished renders out of {} valid renders".format(
len(unfinished_renders), len(valid_renders)))
print("Rendering videos: ")
clean_render_dirs()
# Render videos in multiprocessing queue
multiprocessing.freeze_support()
with multiprocessing.Pool(NUM_MINECRAFT_DIR,
initializer=tqdm.tqdm.set_lock,
initargs=(multiprocessing.RLock(), )) as pool:
manager = ThreadManager(multiprocessing.Manager(), NUM_MINECRAFT_DIR,
0, 1)
func = functools.partial(_render_videos, manager)
num_rendered = list(
tqdm.tqdm(pool.imap_unordered(func, unfinished_renders),
total=len(unfinished_renders),
desc='Files',
miniters=1,
position=0,
maxinterval=1,
smoothing=0))
print('Rendered {} new files!'.format(sum(num_rendered)))
def compute_parallel(self, workers=4):
self._check_mesh_arrays()
mp.freeze_support() # for Windows Support
with mp.Pool(
processes=workers,
initializer=tqdm.set_lock, # WinSup
initargs=(mp.RLock(), )) as pool:
fullstorages = self.meshstor.flatten()
res_cycles = list(
tqdm(pool.imap(self.calculate_single_point, fullstorages),
total=len(fullstorages)))
for flatind, (res, cycles) in enumerate(res_cycles):
ind2d = np.unravel_index(flatind, self.meshstor.shape)
self.meshres[ind2d] = res
self.meshcycle[ind2d] = cycles
def __init__(self, port=9200, host="localhost", version=None) -> None:
super().__init__()
self._port = port
self._host = host
self._version = version or constants.ELASTICSEARCH_DEFAULT_VERSION
self.command_settings = {}
self.directories = self._resolve_directories()
self._elasticsearch_thread = None
self._lifecycle_lock = multiprocessing.RLock()
self._started = False
self._stopped = multiprocessing.Event()
self._starting = multiprocessing.Event()
def __init__(self, N, shape, dtype):
print("SharedCircularBuffer created!")
self._n = N # The number of buffers in the ring
self._dim = np.array(shape) # The shape of each buffer
self._dtype = dtype # The type of the buffer or int bytes
self._index = mp.Value('i',
0) # The index of the next buffer to be put
self._acquired = mp.Value(
'i', -1) # The index of the currently acquired buffer
self._locks = [] # The locks for each buffer in the ring
self._ring = [] # Handles for the shared memory for each buffer
for i in range(N):
arr = mp.sharedctypes.RawArray(dtype, 2 * int(np.prod(self._dim)))
rlock = mp.RLock()
self._ring.append(arr)
self._locks.append(rlock)
def _create_mutable_fields(self, type, max_size):
self._lock = multiprocessing.RLock()
if type == int:
self._value = multiprocessing.Value('i', 0, lock=self._lock)
elif type == long:
self._value = multiprocessing.Value('l', 0L, lock=self._lock)
elif type == float:
self._value = multiprocessing.Value('d', 0.0, lock=self._lock)
elif issubclass(type, basestring):
self._value = multiprocessing.Array('c', max_size, lock=self._lock)
else:
raise ValueError('Type %s is not supported' % type)
self._data_status = multiprocessing.Value('b',
_Option._DATA_DEFAULT,
lock=self._lock)
def __init__(self, maxsize=0):
if not isinstance(maxsize, int):
raise TypeError('maxsize must be type of int')
if maxsize < 0:
raise ValueError('maxsize must be greater than equal to 0')
self._maxsize = maxsize
self._rlock = multiprocessing.RLock()
self._size = multiprocessing.Value(ctypes.c_ulong, 0, lock=self._rlock)
self._pipe_r, self._pipe_w = \
itertools.starmap(
_fdopen,
zip(os.pipe(), ('rb', 'wb'), (0, 0)),
)
def __init__(
self,
observation_space: Space,
action_space: Space,
capacity=int(1e6),
n_step=1,
gamma=0.99,
history_len=1,
mode="numpy",
logdir=None
):
"""
Experience replay buffer for off-policy RL algorithms.
Args:
observation_shape: shape of environment observation
e.g. (8, ) for vector of floats or (84, 84, 3) for RGB image
action_shape: shape of action the agent can take
e.g. (3, ) for 3-dimensional continuous control
capacity: replay buffer capacity
history_len: number of subsequent observations considered a state
n_step: number of time steps between the current state and the next
state in TD backup
gamma: discount factor
discrete actions: True if actions are discrete
"""
# @TODO: Refactor !!!
self.observation_space = observation_space
self.action_space = action_space
self.history_len = history_len
self.capacity = capacity
self.n_step = n_step
self.gamma = gamma
self.len = 0
self.pointer = 0
self._store_lock = mp.RLock()
self.observations, self.actions, self.rewards, self.dones = \
_get_buffers(
capacity=capacity,
observation_space=observation_space,
action_space=action_space,
mode=mode,
logdir=logdir
)
def __init__(
self,
*,
maxsize: int = 1,
use_multiprocessing: bool = False,
time_func: Callable[[], float] = perf_counter,
monitor_thread: bool = False,
):
"""
Initialize the actor system.
:param maxsize: of the message queues
:param use_multiprocessing: set True if you intend to add processes
:param time_func: to measure performance
:param monitor_thread: set True if you want the actor system to request finish when the caller thread finishes
"""
self._maxsize = maxsize
self._mailboxes = OrderedDict()
self._actor_names = set()
self._actors = list()
self._actor_instances = set()
self._multiprocessing = use_multiprocessing
self._started = False
self._enqueued_stop = defaultdict(lambda: 0)
self._enqueued = defaultdict(lambda: 0)
self._dequeued = defaultdict(lambda: 0)
self._time_func = time_func
_shared_lock = multiprocessing.RLock()
self._shared_state = _SharedState(
lock=_shared_lock,
empty=multiprocessing.Condition(lock=_shared_lock),
messages=multiprocessing.Value('l', lock=False),
)
self._collectors = dict()
if monitor_thread:
self._monitor_th = _ActorSystemMonitorThread(
threading.current_thread(), self)
self._monitor_th.start()
else:
self._monitor_th = None
def __init__(self, ser_port, ser_baudrate, last_pos_count):
if last_pos_count < 1:
raise ValueError("last_pos_count shouldn't be less than 1")
self._position_is_fresh = False
self._last_pos_count = last_pos_count
self._last_pos_container = []
self._sync_locker = multiprocessing.RLock()
self._serial = serial.Serial(port=ser_port, baudrate=ser_baudrate)
#self._hot_reset()
self._USBNMEA_OUT()
self._keep_thread_alive = True
self._reader_thread = threading.Thread(target=self._reader_thread_tf,
daemon=True)
self._reader_thread.start()
def rave_pgf_logfile_client(name="RAVE-LOGFILE",
level=LOGLEVEL,
logfile=LOGFILE,
logfilesize=LOGFILESIZE,
nrlogfiles=LOGFILES):
myLogger = logging.getLogger(name)
if not len(myLogger.handlers):
myLogger.setLevel(LOGLEVELS[level])
handler = logging.handlers.RotatingFileHandler(logfile,
maxBytes=logfilesize,
backupCount=nrlogfiles)
# The default formatter contains fractions of a second in the time.
formatter = logging.Formatter(LOGFILE_FORMAT)
handler.setFormatter(formatter)
handler.lock = multiprocessing.RLock()
myLogger.addHandler(handler)
return myLogger
