def test_unlink_semaphore(self):
"""Exercise unlink_semaphore"""
sem = posix_ipc.Semaphore(None, posix_ipc.O_CREX)
posix_ipc.unlink_semaphore(sem.name)
sem.close()
self.assertRaises(posix_ipc.ExistentialError, posix_ipc.Semaphore,
sem.name)
def clean_up(SHARED_MEMORY_NAME, SEMAPHORE_NAME):
try:
posix_ipc.unlink_shared_memory(SHARED_MEMORY_NAME)
posix_ipc.unlink_semaphore(SEMAPHORE_NAME)
except:
print ("memory or semaphore doesn't need cleanup")
print ("\nAll clean!")
def test_unlink_semaphore(self):
"""Exercise unlink_semaphore"""
sem = posix_ipc.Semaphore(None, posix_ipc.O_CREX)
posix_ipc.unlink_semaphore(sem.name)
sem.close()
self.assertRaises(posix_ipc.ExistentialError, posix_ipc.Semaphore,
sem.name)
def __init__(self, control_port, port=None, hwm=10):
self._should_stop = False
self._worker_list = set()
self._need_init = True
if port:
self.init_data(port, hwm)
self._init_control_socket(control_port)
# Cleanup and init global lock and job_uid name ##
self._job_uid = "platoon_{0}_{1}".format(
os.path.basename(os.path.expanduser('~')), control_port)
self._lock_name = "{}lock".format(self._job_uid)
# The ExistentialError is apparently the only way to verify if
# the semaphore/shared_memory exists.
try:
posix_ipc.unlink_semaphore(self._lock_name)
except posix_ipc.ExistentialError:
pass
# Initializing lock
posix_ipc.Semaphore(self._lock_name, posix_ipc.O_CREAT,
initial_value=1)
def _unlink(self):
log.debug("Unlinking trigger semaphore, key='{}'".format(
self._sem_key))
try:
posix_ipc.unlink_semaphore(self._sem_key)
except posix_ipc.ExistentialError:
pass
def __init__(self, control_port, port=None, hwm=10):
self._should_stop = False
self._worker_list = set()
self._need_init = True
if port:
self.init_data(port, hwm)
self._init_control_socket(control_port)
# Cleanup and init global lock and job_uid name ##
self._job_uid = "platoon_{0}_{1}".format(
os.path.basename(os.path.expanduser('~')), control_port)
self._lock_name = "{}lock".format(self._job_uid)
# The ExistentialError is apparently the only way to verify if
# the semaphore/shared_memory exists.
try:
posix_ipc.unlink_semaphore(self._lock_name)
except posix_ipc.ExistentialError:
pass
# Initializing lock
posix_ipc.Semaphore(self._lock_name,
posix_ipc.O_CREAT,
initial_value=1)
def init_shared_params(self,
job_name,
params,
param_sync_rule,
cleanup=False):
"""
Intialize shared memory parameters.
This must be called before accessing the params attribute
and/or calling :meth:`sync_params`, :meth:`lock_params` or
:meth:`unlock_params`.
Paramters
---------
job_name : str
An identifier. This must be the same across all Workers
that share paramters.
params : shared variables
Theano shared variables representing the weights of your model.
param_sync_rule : ParamSyncRule
Update rule for the parameters
cleanup : bool
Whether to cleanup a previous run with the same
identifier. Will also copy the current values of `params`
to the shared memory. This is required on certain
platform due to system restrictions.
"""
self.update_fn = param_sync_rule.make_update_function(params)
if cleanup:
try:
posix_ipc.unlink_semaphore(job_name + 'lock')
except posix_ipc.ExistentialError:
pass
self.lock = posix_ipc.Semaphore(job_name + 'lock',
posix_ipc.O_CREAT,
initial_value=1)
params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape)
for p in params]
params_size = sum(descr_size(*d) for d in params_descr)
if cleanup:
try:
posix_ipc.unlink_shared_memory(job_name + 'params')
except posix_ipc.ExistentialError:
pass
self._shmref = posix_ipc.SharedMemory(job_name + 'params',
posix_ipc.O_CREAT,
size=params_size)
self._shmref = posix_ipc.SharedMemory(job_name + 'params')
self._shm = _mmap(fd=self._shmref.fd, length=params_size)
self._shmref.close_fd()
self.shared_params = []
off = 0
for dtype, shape in params_descr:
self.shared_params.append(
numpy.ndarray(shape, dtype=dtype, buffer=self._shm,
offset=off))
off += descr_size(dtype, shape)
def pyspace_shmem(shmem_name):
nothrow(ExistentialError, unlink_shared_memory, (shmem_name, ))
nothrow(ExistentialError, unlink_semaphore,
('/pyspace_%s_lock' % shmem_name, ))
with pyspaces.PySpaceShMem(shmem_name) as space:
yield space
unlink_shared_memory(shmem_name)
unlink_semaphore('/pyspace_%s_lock' % shmem_name)
def init_shared_params(self, job_name, params, param_sync_rule,
cleanup=False):
"""
Intialize shared memory parameters.
This must be called before accessing the params attribute
and/or calling :meth:`sync_params`, :meth:`lock_params` or
:meth:`unlock_params`.
Paramters
---------
job_name : str
An identifier. This must be the same across all soldiers
that share paramters.
params : shared variables
Theano shared variables representing the weights of your model.
param_sync_rule : ParamSyncRule
Update rule for the parameters
cleanup : bool
Whether to cleanup a previous run with the same
identifier. Will also copy the current values of `params`
to the shared memory. This is required on certain
platform due to system restrictions.
"""
self.local_params = params
self.param_sync_rule = param_sync_rule
if cleanup:
try:
posix_ipc.unlink_semaphore(job_name+'lock')
except posix_ipc.ExistentialError:
pass
self.lock = posix_ipc.Semaphore(job_name+'lock', posix_ipc.O_CREAT,
initial_value=1)
params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape)
for p in params]
params_size = sum(descr_size(*d) for d in params_descr)
if cleanup:
try:
posix_ipc.unlink_shared_memory(job_name+'params')
except posix_ipc.ExistentialError:
pass
self._shmref = posix_ipc.SharedMemory(job_name+'params',
posix_ipc.O_CREAT,
size=params_size)
self._shmref = posix_ipc.SharedMemory(job_name+'params')
self._shm = _mmap(fd=self._shmref.fd, length=params_size)
self._shmref.close_fd()
self.shared_params = []
off = 0
for dtype, shape in params_descr:
self.shared_params.append(numpy.ndarray(shape, dtype=dtype,
buffer=self._shm,
offset=off))
off += descr_size(dtype, shape)
def unlink_all(self):
"""
@brief Unlink (remove) all posix shared memory sections and semaphores.
"""
log.debug("Unlinking all relevant posix shared memory segments and semaphores")
try:
posix_ipc.unlink_semaphore(self.counting_semaphore_key)
except posix_ipc.ExistentialError:
pass
try:
posix_ipc.unlink_semaphore(self.mutex_semaphore_key)
except posix_ipc.ExistentialError:
pass
try:
posix_ipc.unlink_shared_memory(self.shared_buffer_key)
except posix_ipc.ExistentialError:
pass
def __init__(self, name, owner=True):
sem_flags = posix_ipc.O_CREX if owner else 0
if owner:
try:
posix_ipc.unlink_semaphore('/{}_read'.format(name))
except posix_ipc.ExistentialError:
pass
try:
posix_ipc.unlink_semaphore('/{}_write'.format(name))
except posix_ipc.ExistentialError:
pass
self._read_sem = posix_ipc.Semaphore(
'/{}_read'.format(name),
flags=sem_flags,
initial_value=posix_ipc.SEMAPHORE_VALUE_MAX)
self._write_sem = posix_ipc.Semaphore('/{}_write'.format(name),
flags=sem_flags,
initial_value=1)
name = "/" + ''.join(random.sample(NAME_CHARACTERS, NAME_LENGTH))
sem = posix_ipc.Semaphore(name, posix_ipc.O_CREX)
sem.close()
sem.unlink()
print_mem_after()
say("Running semaphore create/destroy test 2...")
print_mem_before()
for i in the_range:
name = "/" + ''.join(random.sample(NAME_CHARACTERS, NAME_LENGTH))
sem = posix_ipc.Semaphore(name, posix_ipc.O_CREX)
sem.close()
posix_ipc.unlink_semaphore(name)
print_mem_after()
say("Running semaphore create/destroy test 3...")
print_mem_before()
for i in the_range:
sem = posix_ipc.Semaphore(None, posix_ipc.O_CREX)
sem.close()
sem.unlink()
print_mem_after()
def __init__(self, control_port=5567, data_port=None, data_hwm=10,
devices=None, experiment_name='',
log_directory='', worker_args='', multi=False):
self._workers = set()
self._need_init = True
self._devices = list()
if devices is not None:
self._devices = Controller.get_workers_devices(devices)
self._local_size = len(self._devices)
self._global_size = self._local_size
self._get_platoon_info_count = [0]
self._init_new_shmem_count = [0]
self._am_i_first_count = [0]
signal.signal(signal.SIGTERM, self._handle_force_close)
signal.signal(signal.SIGINT, self._handle_force_close)
# If we are starting a multi-node training
self._multinode = multi
if self._multinode:
print("## Running in multi-node mode.")
try:
self._init_region_comm()
except Exception as exc:
print("WARNING! {} while being in multi-node mode".format(exc),
file=sys.stderr)
self._region_comm = None
if data_port:
self.init_data(data_port, data_hwm)
self._init_control_socket(control_port)
# Cleanup and init local lock and job_uid name
self._job_uid = "platoon_{0}_{1}".format(
os.path.basename(os.path.expanduser('~')), control_port)
self._lock_name = "{}_lock".format(self._job_uid)
# The ExistentialError is apparently the only way to verify if
# the semaphore/shared_memory exists.
try:
posix_ipc.unlink_semaphore(self._lock_name)
except posix_ipc.ExistentialError:
pass
# Initializing lock
self._lock = posix_ipc.Semaphore(self._lock_name, posix_ipc.O_CREAT,
initial_value=1)
# Init dictionaries for shared buffers with workers
self._shmrefs = dict()
self.shared_buffers = dict()
# Initialize workers
if experiment_name:
try:
os.makedirs(log_directory)
except OSError:
pass
if worker_args is None:
worker_args = ''
worker_args += " --control-port=%d" % control_port
worker_args += " --data-hwm=%d" % data_hwm
if data_port:
worker_args += " --data-port=%d" % data_port
try:
for device in self._devices:
p = launch_process(log_directory, experiment_name,
shlex.split(worker_args), device,
"worker")
self._workers.add(p.pid)
except OSError as exc:
print("ERROR! OS error in Popen: {}".format(exc), file=sys.stderr)
sys.exit(3)
except Exception as exc:
print("ERROR! Other while launching process: {}".format(exc), file=sys.stderr)
sys.exit(4)
import posix_ipc
import utils
params = utils.read_params()
try:
posix_ipc.unlink_shared_memory(params["SHARED_MEMORY_NAME"])
s = "memory segment %s removed" % params["SHARED_MEMORY_NAME"]
print (s)
except:
print ("memory doesn't need cleanup")
try:
posix_ipc.unlink_semaphore(params["SEMAPHORE_NAME"])
s = "semaphore %s removed" % params["SEMAPHORE_NAME"]
print (s)
except:
print ("semaphore doesn't need cleanup")
print ("\nAll clean!")
def posix_unlink():
posix_ipc.unlink_semaphore(self._semaphore1.name)
posix_ipc.unlink_semaphore(self._semaphore2.name)
for shmem_block in self._memory.values():
shmem_block.unlink()
manager.RestartUnit('radiusd.service', 'fail')
except dbus.exceptions.DBusException as err:
print('FR restared skipped ' + str(err))
end = time.time()
logger.info('FR restart ' + str(end - start) + 's.')
Path(RESTART_TIME).touch()
sem_restart_suspended.release()
time.sleep(RESTART_INTERVAL)
sem_restart_suspended.acquire()
logger = init_log()
try:
sem_js = posix_ipc.Semaphore(SEM_JUST_SLEEPING)
posix_ipc.unlink_semaphore(SEM_JUST_SLEEPING)
except:
pass
sem_restart_req = posix_ipc.Semaphore(SEM_RR, posix_ipc.O_CREAT)
sem_restart_suspended = posix_ipc.Semaphore(SEM_JUST_SLEEPING,
posix_ipc.O_CREAT)
if sem_restart_suspended.value == 1:
sem_restart_suspended.acquire()
logger.info("Waiting until semaphore's value > 0")
logger.info("Suspended value " + str(sem_restart_suspended.value))
while True:
sem_restart_req.acquire()
now = datetime.datetime.utcnow()
logger.info('FR restart')
radius_restart()
for i in the_range:
name = "/" + ''.join(random.sample(NAME_CHARACTERS, NAME_LENGTH))
sem = posix_ipc.Semaphore(name, posix_ipc.O_CREX)
sem.unlink()
sem.close()
print_mem_after()
say("Running semaphore create/destroy test 2...")
print_mem_before()
for i in the_range:
name = "/" + ''.join(random.sample(NAME_CHARACTERS, NAME_LENGTH))
sem = posix_ipc.Semaphore(name, posix_ipc.O_CREX)
sem.close()
posix_ipc.unlink_semaphore(name)
print_mem_after()
say("Running semaphore create/destroy test 3...")
print_mem_before()
for i in the_range:
sem = posix_ipc.Semaphore(None, posix_ipc.O_CREX)
sem.unlink()
sem.close()
print_mem_after()
say("Running semaphore create/destroy test 4...")
print_mem_before()
import posix_ipc
# clean up the mess in case sender/receiver was terminated unexpectedly
try:
posix_ipc.unlink_shared_memory("/mysharedmem")
except:
print "failed to clean up shared memory"
try:
posix_ipc.unlink_semaphore("/mysemaphore")
except:
print "failed to clean up semaphore"
import posix_ipc
posix_ipc.unlink_semaphore('/mutex')
posix_ipc.unlink_semaphore('/empty')
posix_ipc.unlink_semaphore('/full')
import posix_ipc
# clean up the mess in case sender/receiver was terminated unexpectedly
try:
posix_ipc.unlink_shared_memory("/mysharedmem")
except:
print "failed to clean up shared memory"
try:
posix_ipc.unlink_semaphore("/mysemaphore")
except:
print "failed to clean up semaphore"
def posix_unlink():
posix_ipc.unlink_semaphore(self._semaphore1.name)
posix_ipc.unlink_semaphore(self._semaphore2.name)
for shmem_block in chain(self._sensors.values(), self._agents.values(), self._settings.values()):
shmem_block.unlink()
def posix_unlink():
posix_ipc.unlink_semaphore("/APP_NAME_SEMAPHORE_SERVER")
posix_ipc.unlink_semaphore("/APP_NAME_SEMAPHORE_CLIENT")
for shmem_block in self._memory.values():
shmem_block.unlink()