def get_zmq_flags(for_cmake=False, for_api=False): r"""Get the necessary flags for compiling & linking with zmq libraries. Args: for_cmake (bool, optional): If True, the returned flags will match the format required by cmake. Defaults to False. for_api (bool, optional): If True, the returned flags will match those required for compiling the API static library. Defaults to False. Returns: tuple(list, list): compile and linker flags. """ _compile_flags = [] _linker_flags = [] # ZMQ library if tools.is_comm_installed('ZMQComm', language='c'): if platform._is_win: # pragma: windows for l in ["libzmq", "czmq"]: plib = ygg_cfg.get('windows', '%s_static' % l, False) pinc = ygg_cfg.get('windows', '%s_include' % l, False) if not (plib and pinc): # pragma: debug raise Exception("Could not locate %s .lib and .h files." % l) pinc_d = os.path.dirname(pinc) plib_d, plib_f = os.path.split(plib) _compile_flags.append("-I%s" % pinc_d) if for_cmake: _linker_flags.append(plib) else: _linker_flags += ['/LIBPATH:%s' % plib_d, plib_f] else: _linker_flags += ["-lczmq", "-lzmq"] _compile_flags += ["-DZMQINSTALLED"] return _compile_flags, _linker_flags
def after_registration(cls): r"""Operations that should be performed to modify class attributes after registration. For compiled languages this includes selecting the default compiler. The order of precedence is the config file 'compiler' option for the language, followed by the environment variable set by _compiler_env, followed by the existing class attribute. """ ModelDriver.after_registration(cls) if cls.language is not None: for k in ['interpreter']: # Set attribute defaults based on config options for k0 in [k, '%s_flags' % k]: ka = 'default_%s' % k0 if k0.endswith('_flags'): old_val = getattr(cls, ka) old_val += ygg_cfg.get(cls.language, k0, '').split() else: setattr( cls, ka, ygg_cfg.get(cls.language, k0, getattr(cls, ka))) # Set default interpreter based on language if cls.default_interpreter is None: cls.default_interpreter = cls.language # Add directory containing the interface cls.paths_to_add.append(cls.get_language_dir())
def test_running(): r"""Test checking for server w/ URL.""" # url = "amqp://*****:*****@localhost:5672/%2F" vhost = ygg_cfg.get('rmq', 'vhost', '') if not vhost: vhost = '%2F' url = "amqp://%s:%s@%s:%s/%s" % (ygg_cfg.get( 'rmq', 'user', 'guest'), ygg_cfg.get('rmq', 'password', 'guest'), ygg_cfg.get('rmq', 'host', 'localhost'), ygg_cfg.get('rmq', 'port', '5672'), vhost) assert (check_rmq_server(url=url))
def check_rmq_server(url=None, **kwargs): r"""Check that connection to a RabbitMQ server is possible. Args: url (str, optional): Address of RMQ server that includes all the necessary information. If this is provided, the remaining arguments are ignored. Defaults to None and the connection parameters are taken from the other arguments. username (str, optional): RMQ server username. Defaults to config value. password (str, optional): RMQ server password. Defaults to config value. host (str, optional): RMQ hostname or IP Address to connect to. Defaults to config value. port (str, optional): RMQ server TCP port to connect to. Defaults to config value. vhost (str, optional): RMQ virtual host to use. Defaults to config value. Returns: bool: True if connection to RabbitMQ server is possible, False otherwise. """ out = True if not _rmq_installed: return False if url is not None: parameters = pika.URLParameters(url) else: username = kwargs.get('username', ygg_cfg.get('rmq', 'user', 'guest')) password = kwargs.get('password', ygg_cfg.get('rmq', 'password', 'guest')) host = kwargs.get('host', ygg_cfg.get('rmq', 'host', _localhost)) port = kwargs.get('port', ygg_cfg.get('rmq', 'port', '5672')) vhost = kwargs.get('vhost', ygg_cfg.get('rmq', 'vhost', '/')) credentials = pika.PlainCredentials(username, password) parameters = pika.ConnectionParameters(host=host, port=int(port), virtual_host=vhost, credentials=credentials) # Try to establish connection logging.getLogger("pika").propagate = False try: from yggdrasil import tools with tools.track_fds("pika test connection: "): connection = pika.BlockingConnection(parameters) if not connection.is_open: # pragma: debug del connection raise BaseException("Connection was not opened.") connection.close() del connection except BaseException as e: # pragma: debug print("Error when attempting to connect to the RabbitMQ server: " + str(e)) out = False logging.getLogger("pika").propagate = True return out
def __init__(self, name, yml=None, env=None, comm_env=None, namespace=None, rank=None, **kwargs): if yml is None: yml = {} if env is None: env = {} if comm_env is None: comm_env = {} # Check if thread initialized to avoid doing it twice for drivers # with multiple inheritance that both need to call __init__ if getattr(self, '_thread_initialized', False): # pragma: debug raise Exception("Thread already initialized. " + "Check multiple inheritance") super(Driver, self).__init__(name, **kwargs) self._thread_initialized = True self.debug('') # if ygg_cfg.get('debug', 'ygg') == 'DEBUG': # self.sleeptime = 1.0 # Set defaults if namespace is None: from yggdrasil.config import ygg_cfg namespace = ygg_cfg.get('rmq', 'namespace') self.debug("Setting namespace to %s", namespace) if kwargs.get('working_dir', None) is None: if isinstance(yml, dict) and ('working_dir' in yml): self.working_dir = yml['working_dir'] else: self.working_dir = os.getcwd() # Assign things self.yml = yml self.env = env self.comm_env = comm_env self.namespace = namespace self.rank = rank self._term_meth = "terminate"
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None, connection_task_method='thread', as_function=False, production_run=False): super(YggRunner, self).__init__('runner') if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') self.namespace = namespace self.host = host self.rank = rank self.connection_task_method = connection_task_method self.modeldrivers = {} self.connectiondrivers = {} self.interrupt_time = 0 self._old_handlers = {} self.production_run = production_run self.error_flag = False self.as_function = as_function self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls self.drivers = yamlfile.parse_yaml(modelYmls, as_function=as_function) self.connectiondrivers = self.drivers['connection'] self.modeldrivers = self.drivers['model']
def get_runner(models, **kwargs): r"""Get runner for a set of models, getting run information from the environment. Args: models (list): List of yaml files containing information on the models that should be run. **kwargs: Additonal keyword arguments are passed to YggRunner. Returns: YggRunner: Runner for the provided models. Raises: Exception: If config option 'namespace' in 'rmq' section not set. """ # Get environment variables logger = logging.getLogger(__name__) namespace = kwargs.pop('namespace', ygg_cfg.get('rmq', 'namespace', False)) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') rank = os.environ.get('PARALLEL_SEQ', '0') host = socket.gethostname() os.environ['YGG_RANK'] = rank os.environ['YGG_HOST'] = host rank = int(rank) kwargs.update(rank=rank, host=host) # Run logger.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) yggRunner = YggRunner(models, namespace, **kwargs) return yggRunner
def build_datatypes(just_obj=False, overwrite=False, as_shared=False): r"""Build the datatypes library.""" if as_shared: dtype_lib = _datatypes_shared_lib else: dtype_lib = _datatypes_static_lib _datatypes_dir = os.path.dirname(dtype_lib) _datatypes_cpp = os.path.join(_datatypes_dir, 'datatypes.cpp') flags = [] pinc = ygg_cfg.get('c', 'rapidjson_include', False) if not pinc: # pragma: debug raise Exception("Could not locate rapidjson include directory.") incl_dir = [_datatypes_dir, _incl_regex, pinc] if platform._is_win: # pragma: windows incl_dir.append(_top_dir) for x in incl_dir: flags += ['-I', x] if platform._is_linux: flags.append('-fPIC') _datatypes_obj = call_compile(_datatypes_cpp, flags=flags, overwrite=overwrite) if just_obj: return _datatypes_obj # Compile regex for windows if platform._is_win: # pragma: windows _regex_obj = build_regex_win32(just_obj=True, overwrite=overwrite) call_link([_regex_obj, _datatypes_obj], dtype_lib, cpp=True, overwrite=overwrite) else: call_link(_datatypes_obj, dtype_lib, cpp=True, overwrite=overwrite)
def start_matlab_engine(skip_connect=False, timeout=None): # pragma: matlab r"""Start a Matlab shared engine session inside a detached screen session. Args: skip_connect (bool, optional): If True, the engine is not connected. Defaults to False. timeout (int, optional): Time (in seconds) that should be waited for Matlab to start up. Defaults to None and is set from the config option ('matlab', 'startup_waittime_s'). Returns: tuple: Information on the started session including the name of the screen session running matlab, the created engine object, the name of the matlab session, and the matlab engine process. Raises: RuntimeError: If Matlab is not installed. """ if not _matlab_engine_installed: # pragma: no matlab raise RuntimeError("Matlab engine is not installed.") if timeout is None: timeout = float(ygg_cfg.get('matlab', 'startup_waittime_s', 10)) old_process = set(locate_matlab_engine_processes()) old_matlab = set(matlab.engine.find_matlab()) screen_session = str('ygg_matlab' + datetime.today().strftime("%Y%j%H%M%S") + '_%d' % len(old_matlab)) try: args = [ 'screen', '-dmS', screen_session, '-c', os.path.join(_top_lang_dir, 'matlab_screenrc'), 'matlab', '-nodisplay', '-nosplash', '-nodesktop', '-nojvm', '-r', '"matlab.engine.shareEngine"' ] subprocess.call(' '.join(args), shell=True) T = TimeOut(timeout) while ((len(set(matlab.engine.find_matlab()) - old_matlab) == 0) and not T.is_out): logger.debug('Waiting for matlab engine to start') sleep(1) # Usually 3 seconds except KeyboardInterrupt: # pragma: debug args = ['screen', '-X', '-S', screen_session, 'quit'] subprocess.call(' '.join(args), shell=True) raise if (len(set(matlab.engine.find_matlab()) - old_matlab) == 0): # pragma: debug raise Exception("start_matlab timed out at %f s" % T.elapsed) new_matlab = list(set(matlab.engine.find_matlab()) - old_matlab)[0] new_process = list(set(locate_matlab_engine_processes()) - old_process)[0] # Connect to the engine matlab_engine = None if not skip_connect: matlab_engine = connect_matlab_engine(new_matlab, first_connect=True) return screen_session, matlab_engine, new_matlab, new_process
def new_comm_kwargs(cls, name, user=None, password=None, host=None, virtual_host=None, port=None, exchange=None, queue='', **kwargs): r"""Initialize communication with new connection. Args: name (str): Name of new connection. user (str, optional): RabbitMQ server username. Defaults to config option 'user' in section 'rmq' if it exists and 'guest' if it does not. password (str, optional): RabbitMQ server password. Defaults to config option 'password' in section 'rmq' if it exists and 'guest' if it does not. host (str, optional): RabbitMQ server host. Defaults to config option 'host' in section 'rmq' if it exists and 'localhost' if it does not. If 'localhost', the output of socket.gethostname() is used. virtual_host (str, optional): RabbitMQ server virtual host. Defaults to config option 'vhost' in section 'rmq' if it exists and '/' if it does not. port (str, optional): Port on host to use. Defaults to config option 'port' in section 'rmq' if it exists and '5672' if it does not. exchange (str, optional): RabbitMQ exchange. Defaults to config option 'namespace' in section 'rmq' if it exits and '' if it does not. queue (str, optional): Name of the queue that messages will be send to or received from. If an empty string, the queue will be a random string and exclusive to a receiving comm. Defaults to ''. **kwargs: Additional keywords arguments are returned as keyword arguments for the new comm. Returns: tuple(tuple, dict): Arguments and keyword arguments for new comm. """ args = [name] if 'address' not in kwargs: if user is None: user = ygg_cfg.get('rmq', 'user', 'guest') if password is None: password = ygg_cfg.get('rmq', 'password', 'guest') if host is None: host = ygg_cfg.get('rmq', 'host', 'localhost') # if host == 'localhost': # host = socket.gethostname() if virtual_host is None: virtual_host = ygg_cfg.get('rmq', 'vhost', '/') if virtual_host == '/': virtual_host = '%2f' if port is None: port = ygg_cfg.get('rmq', 'port', '5672') if exchange is None: exchange = ygg_cfg.get('rmq', 'namespace', '') url = 'amqp://%s:%s@%s:%s/%s' % ( user, password, host, port, virtual_host) kwargs['address'] = _rmq_param_sep.join([url, exchange, queue]) return args, kwargs
def get_rmq_parameters(url=None, user=None, username=None, password=None, host=None, virtual_host=None, vhost=None, port=None, exchange=None, queue=''): r"""Get RabbitMQ connection parameters. Args: url (str, optional): Address of RMQ server that includes all the necessary information. If this is provided, the remaining arguments are ignored. Defaults to None and the connection parameters are taken from the other arguments. user (str, optional): RabbitMQ server username. Defaults to config option 'user' in section 'rmq' if it exists and 'guest' if it does not. username (str, optional): Alias for user. password (str, optional): RabbitMQ server password. Defaults to config option 'password' in section 'rmq' if it exists and 'guest' if it does not. host (str, optional): RabbitMQ server host. Defaults to config option 'host' in section 'rmq' if it exists and _localhost if it does not. If _localhost, the output of socket.gethostname() is used. virtual_host (str, optional): RabbitMQ server virtual host. Defaults to config option 'vhost' in section 'rmq' if it exists and '/' if it does not. vhost (str, optional): Alias for virtual_host. port (str, optional): Port on host to use. Defaults to config option 'port' in section 'rmq' if it exists and '5672' if it does not. exchange (str, optional): RabbitMQ exchange. Defaults to config option 'namespace' in section 'rmq' if it exits and '' if it does not. queue (str, optional): Name of the queue that messages will be send to or received from. If an empty string, the queue will be a random string and exclusive to a receiving comm. Defaults to ''. Returns: tuple: The connection url, exchange, & queue. """ if url is None: if user is None: user = username or ygg_cfg.get('rmq', 'user', 'guest') if password is None: password = ygg_cfg.get('rmq', 'password', 'guest') if host is None: host = ygg_cfg.get('rmq', 'host', _localhost) if virtual_host is None: virtual_host = vhost or ygg_cfg.get('rmq', 'vhost', '/') if virtual_host == '/': virtual_host = '%2f' if port is None: port = ygg_cfg.get('rmq', 'port', '5672') url = 'amqp://%s:%s@%s:%s/%s' % ( user, password, host, port, virtual_host) if exchange is None: exchange = ygg_cfg.get('rmq', 'namespace', '') return url, exchange, queue
def check_czmq(): r"""Determine if the necessary C/C++ libraries are installed for ZeroMQ. Returns: bool: True if the libraries are installed, False otherwise. """ # Check that the libraries are installed if platform._is_win: # pragma: windows opts = [ 'libzmq_include', 'libzmq_static', # 'libzmq_dynamic', 'czmq_include', 'czmq_static' ] # , 'czmq_dynamic'] for o in opts: if not ygg_cfg.get('windows', o, None): # pragma: debug warnings.warn("Config option %s not set." % o) return False return True else: if platform._is_mac: cc = 'clang' else: cc = 'gcc' cc = os.environ.get('CC', cc) try: process = subprocess.Popen([cc, '-lzmq', '-lczmq'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) outs, errs = process.communicate() except OSError: # pragma: debug return False # Python 3 # try: # outs, errs = process.communicate(timeout=15) # except subprocess.TimeoutExpired: # process.kill() # outs, errs = process.communicate() return (b'zmq' not in errs)
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None): super(YggRunner, self).__init__('runner') if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') self.namespace = namespace self.host = host self.rank = rank self.modeldrivers = {} self.inputdrivers = {} self.outputdrivers = {} self.serverdrivers = {} self.interrupt_time = 0 self._inputchannels = {} self._outputchannels = {} self._old_handlers = {} self.error_flag = False self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls drivers = yamlfile.parse_yaml(modelYmls) self.inputdrivers = drivers['input'] self.outputdrivers = drivers['output'] self.modeldrivers = drivers['model'] for x in self.outputdrivers.values(): self._outputchannels[x['args']] = x for x in self.inputdrivers.values(): self._inputchannels[x['args']] = x
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None, connection_task_method='thread', as_service=False, complete_partial=False, partial_commtype=None, production_run=False, mpi_tag_start=None, yaml_param=None, validate=False): self.mpi_comm = None name = 'runner' if MPI is not None: comm = MPI.COMM_WORLD if comm.Get_size() > 1: self.mpi_comm = comm rank = comm.Get_rank() name += str(rank) super(YggRunner, self).__init__(name) if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') if as_service: complete_partial = True self.namespace = namespace self.host = host self.rank = rank self.connection_task_method = connection_task_method self.base_dup = {} self.modelcopies = {} self.modeldrivers = {} self.connectiondrivers = {} self.interrupt_time = 0 self._old_handlers = {} self.production_run = production_run self.error_flag = False self.complete_partial = complete_partial self.partial_commtype = partial_commtype self.validate = validate self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls self.mpi_tag_start = mpi_tag_start if self.mpi_comm and (self.rank > 0): pass else: self.drivers = yamlfile.parse_yaml( modelYmls, complete_partial=complete_partial, partial_commtype=partial_commtype, yaml_param=yaml_param) self.connectiondrivers = self.drivers['connection'] self.modeldrivers = self.drivers['model'] for x in self.modeldrivers.values(): if x['driver'] == 'DummyModelDriver': x['runner'] = self if as_service: for io in x['output_drivers']: for comm in io['inputs']: comm['for_service'] = True for io in x['input_drivers']: for comm in io['outputs']: comm['for_service'] = True
def debug_log(self): # pragma: debug r"""Turn on debugging.""" self._old_loglevel = ygg_cfg.get('debug', 'ygg') ygg_cfg.set('debug', 'ygg', 'DEBUG') cfg_logging()
import subprocess import uuid as uuid_gen import logging from datetime import datetime import os import psutil import warnings import weakref from yggdrasil import backwards, tools, platform, serialize from yggdrasil.languages import get_language_dir from yggdrasil.config import ygg_cfg from yggdrasil.drivers.InterpretedModelDriver import InterpretedModelDriver from yggdrasil.tools import TimeOut, sleep logger = logging.getLogger(__name__) try: # pragma: matlab disable_engine = ygg_cfg.get('matlab', 'disable_engine', 'False').lower() if platform._is_win or (disable_engine == 'true'): _matlab_engine_installed = False if not tools.is_subprocess(): logger.debug("matlab.engine disabled") else: import matlab.engine _matlab_engine_installed = True except ImportError: # pragma: no matlab logger.debug("Could not import matlab.engine. " + "Matlab support for using a sharedEngine will be disabled.") _matlab_engine_installed = False _top_lang_dir = get_language_dir('matlab') _compat_map = {
if platform._is_mac: _shared_ext = '.dylib' else: _shared_ext = '.so' _datatypes_static_lib = os.path.join(_incl_dtype, _prefix + 'datatypes' + _static_ext) _api_static_c = os.path.join(_incl_interface, _prefix + 'ygg' + _static_ext) _api_static_cpp = os.path.join(_incl_interface, _prefix + 'ygg++' + _static_ext) _datatypes_shared_lib = os.path.join(_incl_dtype, _prefix + 'datatypes' + _shared_ext) _api_shared_c = os.path.join(_incl_interface, _prefix + 'ygg' + _shared_ext) _api_shared_cpp = os.path.join(_incl_interface, _prefix + 'ygg++' + _shared_ext) _c_installed = ((len(tools.get_installed_comm(language='c')) > 0) and (ygg_cfg.get('c', 'rapidjson_include', None) is not None)) def get_zmq_flags(for_cmake=False, for_api=False): r"""Get the necessary flags for compiling & linking with zmq libraries. Args: for_cmake (bool, optional): If True, the returned flags will match the format required by cmake. Defaults to False. for_api (bool, optional): If True, the returned flags will match those required for compiling the API static library. Defaults to False. Returns: tuple(list, list): compile and linker flags. """
import traceback import yaml import glob import pprint import functools import threading import logging from yggdrasil import runner from yggdrasil import platform from yggdrasil.multitasking import wait_on_function, ValueEvent from yggdrasil.tools import YggClass, kill from yggdrasil.config import ygg_cfg _service_host_env = 'YGGDRASIL_SERVICE_HOST_URL' _service_repo_dir = 'YGGDRASIL_SERVICE_REPO_DIR' _default_service_type = ygg_cfg.get('services', 'default_type', 'flask') _default_commtype = ygg_cfg.get('services', 'default_comm', None) _default_address = ygg_cfg.get('services', 'address', None) _client_id = ygg_cfg.get('services', 'client_id', None) if platform._is_win: # pragma: windows _shutdown_signal = signal.SIGBREAK else: _shutdown_signal = signal.SIGTERM class ClientError(BaseException): r"""Error raised by errors when calling the server from the client.""" pass class ServerError(BaseException):