def get_zmq_flags(for_cmake=False): r"""Get the necessary flags for compiling & linking with zmq libraries. Args: for_cmake (bool, optional): If True, the returned flags will match the format required by cmake. Defaults to False. Returns: tuple(list, list): compile and linker flags. """ _compile_flags = [] _linker_flags = [] if tools._zmq_installed_c: if platform._is_win: # pragma: windows for l in ["libzmq", "czmq"]: plib = cis_cfg.get('windows', '%s_static' % l, False) pinc = cis_cfg.get('windows', '%s_include' % l, False) if not (plib and pinc): # pragma: debug raise Exception("Could not locate %s .lib and .h files." % l) pinc_d = os.path.dirname(pinc) plib_d, plib_f = os.path.split(plib) _compile_flags.append("-I%s" % pinc_d) if for_cmake: _linker_flags.append(plib) else: _linker_flags += [plib_f, '/LIBPATH:"%s"' % plib_d] else: _linker_flags += ["-lczmq", "-lzmq"] _compile_flags += ["-DZMQINSTALLED"] return _compile_flags, _linker_flags
def check_rmq_server(url=None, **kwargs): r"""Check that connection to a RabbitMQ server is possible. Args: url (str, optional): Address of RMQ server that includes all the necessary information. If this is provided, the remaining arguments are ignored. Defaults to None and the connection parameters are taken from the other arguments. username (str, optional): RMQ server username. Defaults to config value. password (str, optional): RMQ server password. Defaults to config value. host (str, optional): RMQ hostname or IP Address to connect to. Defaults to config value. port (str, optional): RMQ server TCP port to connect to. Defaults to config value. vhost (str, optional): RMQ virtual host to use. Defaults to config value. Returns: bool: True if connection to RabbitMQ server is possible, False otherwise. """ out = True if not _rmq_installed: return False if url is not None: parameters = pika.URLParameters(url) else: username = kwargs.get('username', cis_cfg.get('rmq', 'user', 'guest')) password = kwargs.get('password', cis_cfg.get('rmq', 'password', 'guest')) host = kwargs.get('host', cis_cfg.get('rmq', 'host', 'localhost')) port = kwargs.get('port', cis_cfg.get('rmq', 'port', '5672')) vhost = kwargs.get('vhost', cis_cfg.get('rmq', 'vhost', '/')) credentials = pika.PlainCredentials(username, password) parameters = pika.ConnectionParameters(host=host, port=int(port), virtual_host=vhost, credentials=credentials) # Try to establish connection logging.getLogger("pika").propagate = False try: connection = pika.BlockingConnection(parameters) if not connection.is_open: # pragma: debug raise BaseException("Connection was not openned.") connection.close() except BaseException: # pragma: debug out = False logging.getLogger("pika").propagate = True return out
def get_runner(models, **kwargs): r"""Get runner for a set of models, getting run information from the environment. Args: models (list): List of yaml files containing information on the models that should be run. **kwargs: Additonal keyword arguments are passed to CisRunner. Returns: CisRunner: Runner for the provided models. Raises: Exception: If config option 'namespace' in 'rmq' section not set. """ # Get environment variables logger = logging.getLogger(__name__) namespace = kwargs.pop('namespace', cis_cfg.get('rmq', 'namespace', False)) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') rank = os.environ.get('PARALLEL_SEQ', '0') host = socket.gethostname() os.environ['CIS_RANK'] = rank os.environ['CIS_HOST'] = host rank = int(rank) kwargs.update(rank=rank, host=host) # Run logger.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) cisRunner = CisRunner(models, namespace, **kwargs) return cisRunner
def check_czmq(): r"""Determine if the necessary C/C++ libraries are installed for ZeroMQ. Returns: bool: True if the libraries are installed, False otherwise. """ # Check that the libraries are installed if platform._is_win: # pragma: windows opts = [ 'libzmq_include', 'libzmq_static', # 'libzmq_dynamic', 'czmq_include', 'czmq_static' ] # , 'czmq_dynamic'] for o in opts: if not cis_cfg.get('windows', o, None): # pragma: debug warnings.warn("Config option %s not set." % o) return False return True else: process = subprocess.Popen(['gcc', '-lzmq', '-lczmq'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) outs, errs = process.communicate() # Python 3 # try: # outs, errs = process.communicate(timeout=15) # except subprocess.TimeoutExpired: # process.kill() # outs, errs = process.communicate() return (backwards.unicode2bytes('zmq') not in errs)
def __init__(self, name, yml=None, env=None, comm_env=None, namespace=None, rank=None, **kwargs): if yml is None: yml = {} if env is None: env = {} if comm_env is None: comm_env = {} # Check if thread initialized to avoid doing it twice for drivers # with multiple inheritance that both need to call __init__ if getattr(self, '_thread_initialized', False): # pragma: debug raise Exception("Thread already initialized. " + "Check multiple inheritance") super(Driver, self).__init__(name, **kwargs) self._thread_initialized = True self.debug('') # if cis_cfg.get('debug', 'cis') == 'DEBUG': # self.sleeptime = 1.0 # Set defaults if namespace is None: namespace = cis_cfg.get('rmq', 'namespace') self.debug("Setting namespace to %s", namespace) if kwargs.get('workingDir', None) is None: if isinstance(yml, dict) and ('workingDir' in yml): self.workingDir = yml['workingDir'] else: self.workingDir = os.getcwd() # Assign things self.yml = yml self.env = env self.comm_env = comm_env self.namespace = namespace self.rank = rank self._term_meth = "terminate"
def is_zmq_installed(check_c=True): r"""Determine if the libczmq & libzmq libraries are installed. Returns: bool: True if both libraries are installed, False otherwise. """ # Check existence of zmq python package try: import zmq except ImportError: # pragma: debug return False assert (zmq) if not check_c: # pragma: windows return True # Check existence of config paths for windows if platform._is_win: # pragma: windows opts = [ 'libzmq_include', 'libzmq_static', # 'libzmq_dynamic', 'czmq_include', 'czmq_static' ] # , 'czmq_dynamic'] for o in opts: if not cis_cfg.get('windows', o, None): # pragma: debug warnings.warn("Config option %s not set." % o) return False return True else: process = subprocess.Popen(['gcc', '-lzmq', '-lczmq'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) outs, errs = process.communicate() # Python 3 # try: # outs, errs = process.communicate(timeout=15) # except subprocess.TimeoutExpired: # process.kill() # outs, errs = process.communicate() return (backwards.unicode2bytes('zmq') not in errs)
def new_comm_kwargs(cls, name, user=None, password=None, host=None, virtual_host=None, port=None, exchange=None, queue='', **kwargs): r"""Initialize communication with new connection. Args: name (str): Name of new connection. user (str, optional): RabbitMQ server username. Defaults to config option 'user' in section 'rmq' if it exists and 'guest' if it does not. password (str, optional): RabbitMQ server password. Defaults to config option 'password' in section 'rmq' if it exists and 'guest' if it does not. host (str, optional): RabbitMQ server host. Defaults to config option 'host' in section 'rmq' if it exists and 'localhost' if it does not. If 'localhost', the output of socket.gethostname() is used. virtual_host (str, optional): RabbitMQ server virtual host. Defaults to config option 'vhost' in section 'rmq' if it exists and '/' if it does not. port (str, optional): Port on host to use. Defaults to config option 'port' in section 'rmq' if it exists and '5672' if it does not. exchange (str, optional): RabbitMQ exchange. Defaults to config option 'namespace' in section 'rmq' if it exits and '' if it does not. queue (str, optional): Name of the queue that messages will be send to or received from. If an empty string, the queue will be a random string and exclusive to a receiving comm. Defaults to ''. **kwargs: Additional keywords arguments are returned as keyword arguments for the new comm. Returns: tuple(tuple, dict): Arguments and keyword arguments for new comm. """ args = [name] if 'address' not in kwargs: if user is None: user = cis_cfg.get('rmq', 'user', 'guest') if password is None: password = cis_cfg.get('rmq', 'password', 'guest') if host is None: host = cis_cfg.get('rmq', 'host', 'localhost') # if host == 'localhost': # host = socket.gethostname() if virtual_host is None: virtual_host = cis_cfg.get('rmq', 'vhost', '/') if virtual_host == '/': virtual_host = '%2f' if port is None: port = cis_cfg.get('rmq', 'port', '5672') if exchange is None: exchange = cis_cfg.get('rmq', 'namespace', '') url = 'amqp://%s:%s@%s:%s/%s' % (user, password, host, port, virtual_host) kwargs['address'] = _rmq_param_sep.join([url, exchange, queue]) return args, kwargs
def debug_log(self): # pragma: debug r"""Turn on debugging.""" self._old_loglevel = cis_cfg.get('debug', 'cis') cis_cfg.set('debug', 'cis', 'DEBUG') cfg_logging()