def stop_matlab(screen_session,
matlab_engine,
matlab_session,
matlab_process,
keep_engine=False): # pragma: matlab
r"""Stop a Matlab shared engine session running inside a detached screen
session.
Args:
screen_session (str): Name of the screen session that the shared
Matlab session was started in.
matlab_engine (MatlabEngine): Matlab engine that should be stopped.
matlab_session (str): Name of Matlab session that the Matlab engine is
connected to.
matlab_process (psutil.Process): Process running the Matlab shared engine.
keep_engine (bool, optional): If True, the references to the engine will be
removed so it is not deleted. Defaults to False.
Raises:
RuntimeError: If Matlab is not installed.
"""
if not _matlab_installed: # pragma: no matlab
raise RuntimeError("Matlab is not installed.")
if keep_engine and (matlab_engine is not None):
if '_matlab' in matlab_engine.__dict__:
matlab_engine.quit()
return
# Remove weakrefs to engine to prevent stopping engine more than once
if matlab_engine is not None:
# Remove weak references so engine not deleted on exit
eng_ref = weakref.getweakrefs(matlab_engine)
for x in eng_ref:
if x in matlab.engine._engines:
matlab.engine._engines.remove(x)
# Either exit the engine or remove its reference
if matlab_session in matlab.engine.find_matlab():
try:
matlab_engine.eval('exit', nargout=0)
except BaseException:
pass
else: # pragma: no cover
matlab_engine.__dict__.pop('_matlab', None)
# Stop the screen session containing the Matlab shared session
if screen_session is not None:
if matlab_session in matlab.engine.find_matlab():
os.system(('screen -X -S %s quit') % screen_session)
T = TimeOut(5)
while ((matlab_session in matlab.engine.find_matlab())
and not T.is_out):
debug("Waiting for matlab engine to exit")
sleep(1)
if (matlab_session in matlab.engine.find_matlab()): # pragma: debug
if matlab_process is not None:
matlab_process.terminate()
error("stop_matlab timed out at %f s. " % T.elapsed +
"Killed Matlab sharedEngine process.")
def test_LockedBuffer_clear():
r"""Test the clear method of the LockedBuffer class."""
x = BufferComm.LockedBuffer()
x.append('test')
while x.empty():
tools.sleep(0.1)
assert (not x.empty())
x.clear()
assert_equal(len(x), 0)
def start_matlab(skip_connect=False, timeout=None): # pragma: matlab
r"""Start a Matlab shared engine session inside a detached screen
session.
Args:
skip_connect (bool, optional): If True, the engine is not connected.
Defaults to False.
timeout (int, optional): Time (in seconds) that should be waited for
Matlab to start up. Defaults to None and is set from the config
option ('matlab', 'startup_waittime_s').
Returns:
tuple: Information on the started session including the name of the
screen session running matlab, the created engine object, the name
of the matlab session, and the matlab engine process.
Raises:
RuntimeError: If Matlab is not installed.
"""
if not _matlab_installed: # pragma: no matlab
raise RuntimeError("Matlab is not installed.")
if timeout is None:
timeout = float(config.ygg_cfg.get('matlab', 'startup_waittime_s', 10))
old_process = set(locate_matlab_engine_processes())
old_matlab = set(matlab.engine.find_matlab())
screen_session = str('ygg_matlab' +
datetime.today().strftime("%Y%j%H%M%S") +
'_%d' % len(old_matlab))
try:
args = [
'screen', '-dmS', screen_session, '-c',
os.path.join(os.path.dirname(__file__), 'matlab_screenrc'),
'matlab', '-nodisplay', '-nosplash', '-nodesktop', '-nojvm', '-r',
'"matlab.engine.shareEngine"'
]
subprocess.call(' '.join(args), shell=True)
T = TimeOut(timeout)
while ((len(set(matlab.engine.find_matlab()) - old_matlab) == 0)
and not T.is_out):
debug('Waiting for matlab engine to start')
sleep(1) # Usually 3 seconds
except KeyboardInterrupt: # pragma: debug
args = ['screen', '-X', '-S', screen_session, 'quit']
subprocess.call(' '.join(args), shell=True)
raise
if (len(set(matlab.engine.find_matlab()) -
old_matlab) == 0): # pragma: debug
raise Exception("start_matlab timed out at %f s" % T.elapsed)
new_matlab = list(set(matlab.engine.find_matlab()) - old_matlab)[0]
new_process = list(set(locate_matlab_engine_processes()) - old_process)[0]
# Connect to the engine
matlab_engine = None
if not skip_connect:
matlab_engine = connect_matlab(new_matlab, first_connect=True)
return screen_session, matlab_engine, new_matlab, new_process
def bind_socket(socket, address, retry_timeout=-1, nretry=1):
r"""Bind a socket to an address, getting a random port as necessary.
Args:
socket (zmq.Socket): Socket that should be bound.
address (str): Address that socket should be bound to.
retry_timeout (float, optional): Time (in seconds) that should be
waited before retrying to bind the socket to the address. If
negative, a retry will not be attempted and an error will be
raised. Defaults to -1.
nretry (int, optional): Number of times to try binding the socket to
the addresses. Defaults to 1.
Returns:
str: Address that socket was bound to, including random port if one
was used.
"""
try:
param = parse_address(address)
if (param['protocol'] in ['inproc', 'ipc']) or (param['port']
is not None):
socket.bind(address)
else:
port = socket.bind_to_random_port(address)
address += ":%d" % port
except zmq.ZMQError as e: # pragma: debug
if (retry_timeout < 0) or (nretry == 0):
# if (e.errno not in [48, 98]) or (retry_timeout < 0):
# print(e, e.errno)
raise e
else:
logger.debug("Retrying bind in %f s", retry_timeout)
tools.sleep(retry_timeout)
address = bind_socket(socket,
address,
nretry=nretry - 1,
retry_timeout=retry_timeout)
return address
def fibServer(args):
sleeptime = float(args[0])
print('Hello from Python rpcFibSrv: sleeptime = %f' % sleeptime)
# Create server-side rpc conneciton using model name
rpc = YggRpcServer("rpcFibSrv", "%d", "%d %d")
# Continue receiving requests until error occurs (the connection is closed
# by all clients that have connected).
while True:
print('rpcFibSrv(P): receiving...')
retval, rpc_in = rpc.rpcRecv()
if not retval:
print('rpcFibSrv(P): end of input')
break
# Compute fibonacci number
print('rpcFibSrv(P): <- input %d' % rpc_in[0], end='')
pprev = 0
prev = 1
result = 1
fib_no = 1
arg = rpc_in[0]
while fib_no < arg:
result = prev + pprev
pprev = prev
prev = result
fib_no = fib_no + 1
print(' ::: ->(%2d %2d)' % (arg, result))
# Sleep and then send response back
sleep(float(sleeptime))
flag = rpc.rpcSend(arg, np.int32(result))
if not flag:
raise RuntimeError('rpcFibSrv(P): ERROR sending')
print('Goodbye from Python rpcFibSrv')
def response_loop(cls, client_model, request_id, rpc, time,
internal_variables, external_variables, tables,
table_units, table_lock, synonyms, interpolation,
aggregation):
r"""Check for available data and send response if it is
available.
Args:
client_model (str): Name of model that made the request.
request_id (str): ID associated with request that should
be responded to.
rpc (ServerComm): Server RPC comm that should be used to
reply to the request when the data is available.
time (pandas.Timedelta): Time to get variables at.
internal_variables (list): Variables that model is requesting
that it also calculates.
external_variables (list): Variables that model is requesting
that will be provided by other models.
tables (dict): Mapping from model name to pandas DataFrames
containing variables supplied by the model.
table_units (dict): Mapping from model name to dictionaries
mapping from variable names to units.
table_lock (RLock): Thread-safe lock for accessing table.
synonyms (dict): Dictionary mapping from base variables to
alternate variables and mapping functions used to convert
between the variables. Defaults to empty dict and no
conversions are performed.
interpolation (dict): Mapping from model name to the
interpolation kwargs that should be used. Defaults to
empty dictionary.
aggregation (dict): Mapping from variable name to the
aggregation method that should be used. Defaults to
empty dictionary.
"""
if not (rpc.all_clients_connected and cls.check_for_data(
time, tables, table_units, table_lock, rpc.open_clients)):
# Don't start sampling until all clients have connected
# and there is data available for the requested timestep
tools.sleep(1.0)
return
tot = cls.merge(tables, table_units, table_lock, rpc.open_clients,
synonyms, interpolation, aggregation)
# Update external units
for k in external_variables:
if k not in table_units[client_model]:
table_units[client_model][k] = table_units['base'][k]
# Check if data is available at the desired timestep?
# Convert units
for k in tot.columns:
funits = units.get_conversion_function(
table_units['base'][k], table_units[client_model][k])
tot[k] = tot[k].apply(funits)
# Transform back to variables expected by the model
for kbase, alt in synonyms.get(client_model, {}).items():
if alt['base2alt'] is not None:
alt_vars = alt['base2alt'](tot[kbase])
if isinstance(alt_vars, (tuple, list)):
assert (len(alt_vars) == len(alt['alt']))
for k, v in zip(alt['alt'], alt_vars):
tot[k] = v
else:
assert (len(alt['alt']) == 1)
tot[alt['alt'][0]] = alt_vars
else:
tot[alt['alt'][0]] = tot[kbase]
# Get state
state = {}
for v in internal_variables + external_variables:
v_res = tot.loc[time, v]
state[v] = units.add_units(v_res, table_units[client_model][v])
time_u = units.convert_to(units.convert_from_pandas_timedelta(time),
table_units[client_model]['time'])
flag = rpc.send_to(request_id, state)
if not flag: # pragma: debug
raise RuntimeError(
("Failed to send response to "
"request %s for time %s from "
"model %s.") % (request_id, time_u, client_model))
raise multitasking.BreakLoopException
def target(cls): # pragma: no cover
tools.sleep(10.0)
from yggdrasil.tools import sleep
sleep(1)
print('Python model')
