def get_zmq_flags(for_cmake=False, for_api=False):
r"""Get the necessary flags for compiling & linking with zmq libraries.
Args:
for_cmake (bool, optional): If True, the returned flags will match the
format required by cmake. Defaults to False.
for_api (bool, optional): If True, the returned flags will match those
required for compiling the API static library. Defaults to False.
Returns:
tuple(list, list): compile and linker flags.
"""
_compile_flags = []
_linker_flags = []
# ZMQ library
if tools.is_comm_installed('ZMQComm', language='c'):
if platform._is_win: # pragma: windows
for l in ["libzmq", "czmq"]:
plib = ygg_cfg.get('windows', '%s_static' % l, False)
pinc = ygg_cfg.get('windows', '%s_include' % l, False)
if not (plib and pinc): # pragma: debug
raise Exception("Could not locate %s .lib and .h files." %
l)
pinc_d = os.path.dirname(pinc)
plib_d, plib_f = os.path.split(plib)
_compile_flags.append("-I%s" % pinc_d)
if for_cmake:
_linker_flags.append(plib)
else:
_linker_flags += ['/LIBPATH:%s' % plib_d, plib_f]
else:
_linker_flags += ["-lczmq", "-lzmq"]
_compile_flags += ["-DZMQINSTALLED"]
return _compile_flags, _linker_flags
def after_registration(cls):
r"""Operations that should be performed to modify class attributes after
registration. For compiled languages this includes selecting the
default compiler. The order of precedence is the config file 'compiler'
option for the language, followed by the environment variable set by
_compiler_env, followed by the existing class attribute.
"""
ModelDriver.after_registration(cls)
if cls.language is not None:
for k in ['interpreter']:
# Set attribute defaults based on config options
for k0 in [k, '%s_flags' % k]:
ka = 'default_%s' % k0
if k0.endswith('_flags'):
old_val = getattr(cls, ka)
old_val += ygg_cfg.get(cls.language, k0, '').split()
else:
setattr(
cls, ka,
ygg_cfg.get(cls.language, k0, getattr(cls, ka)))
# Set default interpreter based on language
if cls.default_interpreter is None:
cls.default_interpreter = cls.language
# Add directory containing the interface
cls.paths_to_add.append(cls.get_language_dir())
def test_running():
r"""Test checking for server w/ URL."""
# url = "amqp://*****:*****@localhost:5672/%2F"
vhost = ygg_cfg.get('rmq', 'vhost', '')
if not vhost:
vhost = '%2F'
url = "amqp://%s:%s@%s:%s/%s" % (ygg_cfg.get(
'rmq', 'user', 'guest'), ygg_cfg.get('rmq', 'password', 'guest'),
ygg_cfg.get('rmq', 'host', 'localhost'),
ygg_cfg.get('rmq', 'port', '5672'), vhost)
assert (check_rmq_server(url=url))
def check_rmq_server(url=None, **kwargs):
r"""Check that connection to a RabbitMQ server is possible.
Args:
url (str, optional): Address of RMQ server that includes all the
necessary information. If this is provided, the remaining arguments
are ignored. Defaults to None and the connection parameters are
taken from the other arguments.
username (str, optional): RMQ server username. Defaults to config value.
password (str, optional): RMQ server password. Defaults to config value.
host (str, optional): RMQ hostname or IP Address to connect to. Defaults
to config value.
port (str, optional): RMQ server TCP port to connect to. Defaults to
config value.
vhost (str, optional): RMQ virtual host to use. Defaults to config value.
Returns:
bool: True if connection to RabbitMQ server is possible, False
otherwise.
"""
out = True
if not _rmq_installed:
return False
if url is not None:
parameters = pika.URLParameters(url)
else:
username = kwargs.get('username', ygg_cfg.get('rmq', 'user', 'guest'))
password = kwargs.get('password', ygg_cfg.get('rmq', 'password', 'guest'))
host = kwargs.get('host', ygg_cfg.get('rmq', 'host', _localhost))
port = kwargs.get('port', ygg_cfg.get('rmq', 'port', '5672'))
vhost = kwargs.get('vhost', ygg_cfg.get('rmq', 'vhost', '/'))
credentials = pika.PlainCredentials(username, password)
parameters = pika.ConnectionParameters(host=host, port=int(port),
virtual_host=vhost,
credentials=credentials)
# Try to establish connection
logging.getLogger("pika").propagate = False
try:
from yggdrasil import tools
with tools.track_fds("pika test connection: "):
connection = pika.BlockingConnection(parameters)
if not connection.is_open: # pragma: debug
del connection
raise BaseException("Connection was not opened.")
connection.close()
del connection
except BaseException as e: # pragma: debug
print("Error when attempting to connect to the RabbitMQ server: "
+ str(e))
out = False
logging.getLogger("pika").propagate = True
return out
def __init__(self, name, yml=None, env=None, comm_env=None, namespace=None,
rank=None, **kwargs):
if yml is None:
yml = {}
if env is None:
env = {}
if comm_env is None:
comm_env = {}
# Check if thread initialized to avoid doing it twice for drivers
# with multiple inheritance that both need to call __init__
if getattr(self, '_thread_initialized', False): # pragma: debug
raise Exception("Thread already initialized. "
+ "Check multiple inheritance")
super(Driver, self).__init__(name, **kwargs)
self._thread_initialized = True
self.debug('')
# if ygg_cfg.get('debug', 'ygg') == 'DEBUG':
# self.sleeptime = 1.0
# Set defaults
if namespace is None:
from yggdrasil.config import ygg_cfg
namespace = ygg_cfg.get('rmq', 'namespace')
self.debug("Setting namespace to %s", namespace)
if kwargs.get('working_dir', None) is None:
if isinstance(yml, dict) and ('working_dir' in yml):
self.working_dir = yml['working_dir']
else:
self.working_dir = os.getcwd()
# Assign things
self.yml = yml
self.env = env
self.comm_env = comm_env
self.namespace = namespace
self.rank = rank
self._term_meth = "terminate"
def __init__(self,
modelYmls,
namespace=None,
host=None,
rank=0,
ygg_debug_level=None,
rmq_debug_level=None,
ygg_debug_prefix=None,
connection_task_method='thread',
as_function=False,
production_run=False):
super(YggRunner, self).__init__('runner')
if namespace is None:
namespace = ygg_cfg.get('rmq', 'namespace', False)
if not namespace: # pragma: debug
raise Exception('rmq:namespace not set in config file')
self.namespace = namespace
self.host = host
self.rank = rank
self.connection_task_method = connection_task_method
self.modeldrivers = {}
self.connectiondrivers = {}
self.interrupt_time = 0
self._old_handlers = {}
self.production_run = production_run
self.error_flag = False
self.as_function = as_function
self.debug("Running in %s with path %s namespace %s rank %d",
os.getcwd(), sys.path, namespace, rank)
# Update environment based on config
cfg_environment()
# Parse yamls
self.drivers = yamlfile.parse_yaml(modelYmls, as_function=as_function)
self.connectiondrivers = self.drivers['connection']
self.modeldrivers = self.drivers['model']
def get_runner(models, **kwargs):
r"""Get runner for a set of models, getting run information from the
environment.
Args:
models (list): List of yaml files containing information on the models
that should be run.
**kwargs: Additonal keyword arguments are passed to YggRunner.
Returns:
YggRunner: Runner for the provided models.
Raises:
Exception: If config option 'namespace' in 'rmq' section not set.
"""
# Get environment variables
logger = logging.getLogger(__name__)
namespace = kwargs.pop('namespace', ygg_cfg.get('rmq', 'namespace', False))
if not namespace: # pragma: debug
raise Exception('rmq:namespace not set in config file')
rank = os.environ.get('PARALLEL_SEQ', '0')
host = socket.gethostname()
os.environ['YGG_RANK'] = rank
os.environ['YGG_HOST'] = host
rank = int(rank)
kwargs.update(rank=rank, host=host)
# Run
logger.debug("Running in %s with path %s namespace %s rank %d",
os.getcwd(), sys.path, namespace, rank)
yggRunner = YggRunner(models, namespace, **kwargs)
return yggRunner
def build_datatypes(just_obj=False, overwrite=False, as_shared=False):
r"""Build the datatypes library."""
if as_shared:
dtype_lib = _datatypes_shared_lib
else:
dtype_lib = _datatypes_static_lib
_datatypes_dir = os.path.dirname(dtype_lib)
_datatypes_cpp = os.path.join(_datatypes_dir, 'datatypes.cpp')
flags = []
pinc = ygg_cfg.get('c', 'rapidjson_include', False)
if not pinc: # pragma: debug
raise Exception("Could not locate rapidjson include directory.")
incl_dir = [_datatypes_dir, _incl_regex, pinc]
if platform._is_win: # pragma: windows
incl_dir.append(_top_dir)
for x in incl_dir:
flags += ['-I', x]
if platform._is_linux:
flags.append('-fPIC')
_datatypes_obj = call_compile(_datatypes_cpp,
flags=flags,
overwrite=overwrite)
if just_obj:
return _datatypes_obj
# Compile regex for windows
if platform._is_win: # pragma: windows
_regex_obj = build_regex_win32(just_obj=True, overwrite=overwrite)
call_link([_regex_obj, _datatypes_obj],
dtype_lib,
cpp=True,
overwrite=overwrite)
else:
call_link(_datatypes_obj, dtype_lib, cpp=True, overwrite=overwrite)
def start_matlab_engine(skip_connect=False, timeout=None): # pragma: matlab
r"""Start a Matlab shared engine session inside a detached screen
session.
Args:
skip_connect (bool, optional): If True, the engine is not connected.
Defaults to False.
timeout (int, optional): Time (in seconds) that should be waited for
Matlab to start up. Defaults to None and is set from the config
option ('matlab', 'startup_waittime_s').
Returns:
tuple: Information on the started session including the name of the
screen session running matlab, the created engine object, the name
of the matlab session, and the matlab engine process.
Raises:
RuntimeError: If Matlab is not installed.
"""
if not _matlab_engine_installed: # pragma: no matlab
raise RuntimeError("Matlab engine is not installed.")
if timeout is None:
timeout = float(ygg_cfg.get('matlab', 'startup_waittime_s', 10))
old_process = set(locate_matlab_engine_processes())
old_matlab = set(matlab.engine.find_matlab())
screen_session = str('ygg_matlab' +
datetime.today().strftime("%Y%j%H%M%S") +
'_%d' % len(old_matlab))
try:
args = [
'screen', '-dmS', screen_session, '-c',
os.path.join(_top_lang_dir, 'matlab_screenrc'), 'matlab',
'-nodisplay', '-nosplash', '-nodesktop', '-nojvm', '-r',
'"matlab.engine.shareEngine"'
]
subprocess.call(' '.join(args), shell=True)
T = TimeOut(timeout)
while ((len(set(matlab.engine.find_matlab()) - old_matlab) == 0)
and not T.is_out):
logger.debug('Waiting for matlab engine to start')
sleep(1) # Usually 3 seconds
except KeyboardInterrupt: # pragma: debug
args = ['screen', '-X', '-S', screen_session, 'quit']
subprocess.call(' '.join(args), shell=True)
raise
if (len(set(matlab.engine.find_matlab()) -
old_matlab) == 0): # pragma: debug
raise Exception("start_matlab timed out at %f s" % T.elapsed)
new_matlab = list(set(matlab.engine.find_matlab()) - old_matlab)[0]
new_process = list(set(locate_matlab_engine_processes()) - old_process)[0]
# Connect to the engine
matlab_engine = None
if not skip_connect:
matlab_engine = connect_matlab_engine(new_matlab, first_connect=True)
return screen_session, matlab_engine, new_matlab, new_process
def new_comm_kwargs(cls, name, user=None, password=None, host=None,
virtual_host=None, port=None, exchange=None, queue='',
**kwargs):
r"""Initialize communication with new connection.
Args:
name (str): Name of new connection.
user (str, optional): RabbitMQ server username. Defaults to config
option 'user' in section 'rmq' if it exists and 'guest' if it
does not.
password (str, optional): RabbitMQ server password. Defaults to
config option 'password' in section 'rmq' if it exists and
'guest' if it does not.
host (str, optional): RabbitMQ server host. Defaults to config option
'host' in section 'rmq' if it exists and 'localhost' if it
does not. If 'localhost', the output of socket.gethostname()
is used.
virtual_host (str, optional): RabbitMQ server virtual host. Defaults
to config option 'vhost' in section 'rmq' if it exists and '/'
if it does not.
port (str, optional): Port on host to use. Defaults to config option
'port' in section 'rmq' if it exists and '5672' if it does not.
exchange (str, optional): RabbitMQ exchange. Defaults to config
option 'namespace' in section 'rmq' if it exits and '' if it does
not.
queue (str, optional): Name of the queue that messages will be
send to or received from. If an empty string, the queue will
be a random string and exclusive to a receiving comm. Defaults
to ''.
**kwargs: Additional keywords arguments are returned as keyword
arguments for the new comm.
Returns:
tuple(tuple, dict): Arguments and keyword arguments for new comm.
"""
args = [name]
if 'address' not in kwargs:
if user is None:
user = ygg_cfg.get('rmq', 'user', 'guest')
if password is None:
password = ygg_cfg.get('rmq', 'password', 'guest')
if host is None:
host = ygg_cfg.get('rmq', 'host', 'localhost')
# if host == 'localhost':
# host = socket.gethostname()
if virtual_host is None:
virtual_host = ygg_cfg.get('rmq', 'vhost', '/')
if virtual_host == '/':
virtual_host = '%2f'
if port is None:
port = ygg_cfg.get('rmq', 'port', '5672')
if exchange is None:
exchange = ygg_cfg.get('rmq', 'namespace', '')
url = 'amqp://%s:%s@%s:%s/%s' % (
user, password, host, port, virtual_host)
kwargs['address'] = _rmq_param_sep.join([url, exchange, queue])
return args, kwargs
def get_rmq_parameters(url=None, user=None, username=None,
password=None, host=None, virtual_host=None,
vhost=None, port=None, exchange=None, queue=''):
r"""Get RabbitMQ connection parameters.
Args:
url (str, optional): Address of RMQ server that includes all the
necessary information. If this is provided, the remaining arguments
are ignored. Defaults to None and the connection parameters are
taken from the other arguments.
user (str, optional): RabbitMQ server username. Defaults to config
option 'user' in section 'rmq' if it exists and 'guest' if it
does not.
username (str, optional): Alias for user.
password (str, optional): RabbitMQ server password. Defaults to
config option 'password' in section 'rmq' if it exists and
'guest' if it does not.
host (str, optional): RabbitMQ server host. Defaults to config option
'host' in section 'rmq' if it exists and _localhost if it
does not. If _localhost, the output of socket.gethostname()
is used.
virtual_host (str, optional): RabbitMQ server virtual host. Defaults
to config option 'vhost' in section 'rmq' if it exists and '/'
if it does not.
vhost (str, optional): Alias for virtual_host.
port (str, optional): Port on host to use. Defaults to config option
'port' in section 'rmq' if it exists and '5672' if it does not.
exchange (str, optional): RabbitMQ exchange. Defaults to config
option 'namespace' in section 'rmq' if it exits and '' if it does
not.
queue (str, optional): Name of the queue that messages will be
send to or received from. If an empty string, the queue will
be a random string and exclusive to a receiving comm. Defaults
to ''.
Returns:
tuple: The connection url, exchange, & queue.
"""
if url is None:
if user is None:
user = username or ygg_cfg.get('rmq', 'user', 'guest')
if password is None:
password = ygg_cfg.get('rmq', 'password', 'guest')
if host is None:
host = ygg_cfg.get('rmq', 'host', _localhost)
if virtual_host is None:
virtual_host = vhost or ygg_cfg.get('rmq', 'vhost', '/')
if virtual_host == '/':
virtual_host = '%2f'
if port is None:
port = ygg_cfg.get('rmq', 'port', '5672')
url = 'amqp://%s:%s@%s:%s/%s' % (
user, password, host, port, virtual_host)
if exchange is None:
exchange = ygg_cfg.get('rmq', 'namespace', '')
return url, exchange, queue
def check_czmq():
r"""Determine if the necessary C/C++ libraries are installed for ZeroMQ.
Returns:
bool: True if the libraries are installed, False otherwise.
"""
# Check that the libraries are installed
if platform._is_win: # pragma: windows
opts = [
'libzmq_include',
'libzmq_static', # 'libzmq_dynamic',
'czmq_include',
'czmq_static'
] # , 'czmq_dynamic']
for o in opts:
if not ygg_cfg.get('windows', o, None): # pragma: debug
warnings.warn("Config option %s not set." % o)
return False
return True
else:
if platform._is_mac:
cc = 'clang'
else:
cc = 'gcc'
cc = os.environ.get('CC', cc)
try:
process = subprocess.Popen([cc, '-lzmq', '-lczmq'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outs, errs = process.communicate()
except OSError: # pragma: debug
return False
# Python 3
# try:
# outs, errs = process.communicate(timeout=15)
# except subprocess.TimeoutExpired:
# process.kill()
# outs, errs = process.communicate()
return (b'zmq' not in errs)
def __init__(self,
modelYmls,
namespace=None,
host=None,
rank=0,
ygg_debug_level=None,
rmq_debug_level=None,
ygg_debug_prefix=None):
super(YggRunner, self).__init__('runner')
if namespace is None:
namespace = ygg_cfg.get('rmq', 'namespace', False)
if not namespace: # pragma: debug
raise Exception('rmq:namespace not set in config file')
self.namespace = namespace
self.host = host
self.rank = rank
self.modeldrivers = {}
self.inputdrivers = {}
self.outputdrivers = {}
self.serverdrivers = {}
self.interrupt_time = 0
self._inputchannels = {}
self._outputchannels = {}
self._old_handlers = {}
self.error_flag = False
self.debug("Running in %s with path %s namespace %s rank %d",
os.getcwd(), sys.path, namespace, rank)
# Update environment based on config
cfg_environment()
# Parse yamls
drivers = yamlfile.parse_yaml(modelYmls)
self.inputdrivers = drivers['input']
self.outputdrivers = drivers['output']
self.modeldrivers = drivers['model']
for x in self.outputdrivers.values():
self._outputchannels[x['args']] = x
for x in self.inputdrivers.values():
self._inputchannels[x['args']] = x
def __init__(self,
modelYmls,
namespace=None,
host=None,
rank=0,
ygg_debug_level=None,
rmq_debug_level=None,
ygg_debug_prefix=None,
connection_task_method='thread',
as_service=False,
complete_partial=False,
partial_commtype=None,
production_run=False,
mpi_tag_start=None,
yaml_param=None,
validate=False):
self.mpi_comm = None
name = 'runner'
if MPI is not None:
comm = MPI.COMM_WORLD
if comm.Get_size() > 1:
self.mpi_comm = comm
rank = comm.Get_rank()
name += str(rank)
super(YggRunner, self).__init__(name)
if namespace is None:
namespace = ygg_cfg.get('rmq', 'namespace', False)
if not namespace: # pragma: debug
raise Exception('rmq:namespace not set in config file')
if as_service:
complete_partial = True
self.namespace = namespace
self.host = host
self.rank = rank
self.connection_task_method = connection_task_method
self.base_dup = {}
self.modelcopies = {}
self.modeldrivers = {}
self.connectiondrivers = {}
self.interrupt_time = 0
self._old_handlers = {}
self.production_run = production_run
self.error_flag = False
self.complete_partial = complete_partial
self.partial_commtype = partial_commtype
self.validate = validate
self.debug("Running in %s with path %s namespace %s rank %d",
os.getcwd(), sys.path, namespace, rank)
# Update environment based on config
cfg_environment()
# Parse yamls
self.mpi_tag_start = mpi_tag_start
if self.mpi_comm and (self.rank > 0):
pass
else:
self.drivers = yamlfile.parse_yaml(
modelYmls,
complete_partial=complete_partial,
partial_commtype=partial_commtype,
yaml_param=yaml_param)
self.connectiondrivers = self.drivers['connection']
self.modeldrivers = self.drivers['model']
for x in self.modeldrivers.values():
if x['driver'] == 'DummyModelDriver':
x['runner'] = self
if as_service:
for io in x['output_drivers']:
for comm in io['inputs']:
comm['for_service'] = True
for io in x['input_drivers']:
for comm in io['outputs']:
comm['for_service'] = True
def debug_log(self): # pragma: debug
r"""Turn on debugging."""
self._old_loglevel = ygg_cfg.get('debug', 'ygg')
ygg_cfg.set('debug', 'ygg', 'DEBUG')
cfg_logging()
import subprocess
import uuid as uuid_gen
import logging
from datetime import datetime
import os
import psutil
import warnings
import weakref
from yggdrasil import backwards, tools, platform, serialize
from yggdrasil.languages import get_language_dir
from yggdrasil.config import ygg_cfg
from yggdrasil.drivers.InterpretedModelDriver import InterpretedModelDriver
from yggdrasil.tools import TimeOut, sleep
logger = logging.getLogger(__name__)
try: # pragma: matlab
disable_engine = ygg_cfg.get('matlab', 'disable_engine', 'False').lower()
if platform._is_win or (disable_engine == 'true'):
_matlab_engine_installed = False
if not tools.is_subprocess():
logger.debug("matlab.engine disabled")
else:
import matlab.engine
_matlab_engine_installed = True
except ImportError: # pragma: no matlab
logger.debug("Could not import matlab.engine. "
+ "Matlab support for using a sharedEngine will be disabled.")
_matlab_engine_installed = False
_top_lang_dir = get_language_dir('matlab')
_compat_map = {
if platform._is_mac:
_shared_ext = '.dylib'
else:
_shared_ext = '.so'
_datatypes_static_lib = os.path.join(_incl_dtype,
_prefix + 'datatypes' + _static_ext)
_api_static_c = os.path.join(_incl_interface, _prefix + 'ygg' + _static_ext)
_api_static_cpp = os.path.join(_incl_interface,
_prefix + 'ygg++' + _static_ext)
_datatypes_shared_lib = os.path.join(_incl_dtype,
_prefix + 'datatypes' + _shared_ext)
_api_shared_c = os.path.join(_incl_interface, _prefix + 'ygg' + _shared_ext)
_api_shared_cpp = os.path.join(_incl_interface,
_prefix + 'ygg++' + _shared_ext)
_c_installed = ((len(tools.get_installed_comm(language='c')) > 0)
and (ygg_cfg.get('c', 'rapidjson_include', None) is not None))
def get_zmq_flags(for_cmake=False, for_api=False):
r"""Get the necessary flags for compiling & linking with zmq libraries.
Args:
for_cmake (bool, optional): If True, the returned flags will match the
format required by cmake. Defaults to False.
for_api (bool, optional): If True, the returned flags will match those
required for compiling the API static library. Defaults to False.
Returns:
tuple(list, list): compile and linker flags.
"""
import traceback
import yaml
import glob
import pprint
import functools
import threading
import logging
from yggdrasil import runner
from yggdrasil import platform
from yggdrasil.multitasking import wait_on_function, ValueEvent
from yggdrasil.tools import YggClass, kill
from yggdrasil.config import ygg_cfg
_service_host_env = 'YGGDRASIL_SERVICE_HOST_URL'
_service_repo_dir = 'YGGDRASIL_SERVICE_REPO_DIR'
_default_service_type = ygg_cfg.get('services', 'default_type', 'flask')
_default_commtype = ygg_cfg.get('services', 'default_comm', None)
_default_address = ygg_cfg.get('services', 'address', None)
_client_id = ygg_cfg.get('services', 'client_id', None)
if platform._is_win: # pragma: windows
_shutdown_signal = signal.SIGBREAK
else:
_shutdown_signal = signal.SIGTERM
class ClientError(BaseException):
r"""Error raised by errors when calling the server from the client."""
pass
class ServerError(BaseException):
