def get_zmq_flags(for_cmake=False):
r"""Get the necessary flags for compiling & linking with zmq libraries.
Args:
for_cmake (bool, optional): If True, the returned flags will match the
format required by cmake. Defaults to False.
Returns:
tuple(list, list): compile and linker flags.
"""
_compile_flags = []
_linker_flags = []
if tools._zmq_installed_c:
if platform._is_win: # pragma: windows
for l in ["libzmq", "czmq"]:
plib = cis_cfg.get('windows', '%s_static' % l, False)
pinc = cis_cfg.get('windows', '%s_include' % l, False)
if not (plib and pinc): # pragma: debug
raise Exception("Could not locate %s .lib and .h files." %
l)
pinc_d = os.path.dirname(pinc)
plib_d, plib_f = os.path.split(plib)
_compile_flags.append("-I%s" % pinc_d)
if for_cmake:
_linker_flags.append(plib)
else:
_linker_flags += [plib_f, '/LIBPATH:"%s"' % plib_d]
else:
_linker_flags += ["-lczmq", "-lzmq"]
_compile_flags += ["-DZMQINSTALLED"]
return _compile_flags, _linker_flags
def check_rmq_server(url=None, **kwargs):
r"""Check that connection to a RabbitMQ server is possible.
Args:
url (str, optional): Address of RMQ server that includes all the
necessary information. If this is provided, the remaining arguments
are ignored. Defaults to None and the connection parameters are
taken from the other arguments.
username (str, optional): RMQ server username. Defaults to config value.
password (str, optional): RMQ server password. Defaults to config value.
host (str, optional): RMQ hostname or IP Address to connect to. Defaults
to config value.
port (str, optional): RMQ server TCP port to connect to. Defaults to
config value.
vhost (str, optional): RMQ virtual host to use. Defaults to config value.
Returns:
bool: True if connection to RabbitMQ server is possible, False
otherwise.
"""
out = True
if not _rmq_installed:
return False
if url is not None:
parameters = pika.URLParameters(url)
else:
username = kwargs.get('username', cis_cfg.get('rmq', 'user', 'guest'))
password = kwargs.get('password',
cis_cfg.get('rmq', 'password', 'guest'))
host = kwargs.get('host', cis_cfg.get('rmq', 'host', 'localhost'))
port = kwargs.get('port', cis_cfg.get('rmq', 'port', '5672'))
vhost = kwargs.get('vhost', cis_cfg.get('rmq', 'vhost', '/'))
credentials = pika.PlainCredentials(username, password)
parameters = pika.ConnectionParameters(host=host,
port=int(port),
virtual_host=vhost,
credentials=credentials)
# Try to establish connection
logging.getLogger("pika").propagate = False
try:
connection = pika.BlockingConnection(parameters)
if not connection.is_open: # pragma: debug
raise BaseException("Connection was not openned.")
connection.close()
except BaseException: # pragma: debug
out = False
logging.getLogger("pika").propagate = True
return out
def get_runner(models, **kwargs):
r"""Get runner for a set of models, getting run information from the
environment.
Args:
models (list): List of yaml files containing information on the models
that should be run.
**kwargs: Additonal keyword arguments are passed to CisRunner.
Returns:
CisRunner: Runner for the provided models.
Raises:
Exception: If config option 'namespace' in 'rmq' section not set.
"""
# Get environment variables
logger = logging.getLogger(__name__)
namespace = kwargs.pop('namespace', cis_cfg.get('rmq', 'namespace', False))
if not namespace: # pragma: debug
raise Exception('rmq:namespace not set in config file')
rank = os.environ.get('PARALLEL_SEQ', '0')
host = socket.gethostname()
os.environ['CIS_RANK'] = rank
os.environ['CIS_HOST'] = host
rank = int(rank)
kwargs.update(rank=rank, host=host)
# Run
logger.debug("Running in %s with path %s namespace %s rank %d",
os.getcwd(), sys.path, namespace, rank)
cisRunner = CisRunner(models, namespace, **kwargs)
return cisRunner
def check_czmq():
r"""Determine if the necessary C/C++ libraries are installed for ZeroMQ.
Returns:
bool: True if the libraries are installed, False otherwise.
"""
# Check that the libraries are installed
if platform._is_win: # pragma: windows
opts = [
'libzmq_include',
'libzmq_static', # 'libzmq_dynamic',
'czmq_include',
'czmq_static'
] # , 'czmq_dynamic']
for o in opts:
if not cis_cfg.get('windows', o, None): # pragma: debug
warnings.warn("Config option %s not set." % o)
return False
return True
else:
process = subprocess.Popen(['gcc', '-lzmq', '-lczmq'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outs, errs = process.communicate()
# Python 3
# try:
# outs, errs = process.communicate(timeout=15)
# except subprocess.TimeoutExpired:
# process.kill()
# outs, errs = process.communicate()
return (backwards.unicode2bytes('zmq') not in errs)
def __init__(self, name, yml=None, env=None, comm_env=None, namespace=None,
rank=None, **kwargs):
if yml is None:
yml = {}
if env is None:
env = {}
if comm_env is None:
comm_env = {}
# Check if thread initialized to avoid doing it twice for drivers
# with multiple inheritance that both need to call __init__
if getattr(self, '_thread_initialized', False): # pragma: debug
raise Exception("Thread already initialized. " +
"Check multiple inheritance")
super(Driver, self).__init__(name, **kwargs)
self._thread_initialized = True
self.debug('')
# if cis_cfg.get('debug', 'cis') == 'DEBUG':
# self.sleeptime = 1.0
# Set defaults
if namespace is None:
namespace = cis_cfg.get('rmq', 'namespace')
self.debug("Setting namespace to %s", namespace)
if kwargs.get('workingDir', None) is None:
if isinstance(yml, dict) and ('workingDir' in yml):
self.workingDir = yml['workingDir']
else:
self.workingDir = os.getcwd()
# Assign things
self.yml = yml
self.env = env
self.comm_env = comm_env
self.namespace = namespace
self.rank = rank
self._term_meth = "terminate"
def is_zmq_installed(check_c=True):
r"""Determine if the libczmq & libzmq libraries are installed.
Returns:
bool: True if both libraries are installed, False otherwise.
"""
# Check existence of zmq python package
try:
import zmq
except ImportError: # pragma: debug
return False
assert (zmq)
if not check_c: # pragma: windows
return True
# Check existence of config paths for windows
if platform._is_win: # pragma: windows
opts = [
'libzmq_include',
'libzmq_static', # 'libzmq_dynamic',
'czmq_include',
'czmq_static'
] # , 'czmq_dynamic']
for o in opts:
if not cis_cfg.get('windows', o, None): # pragma: debug
warnings.warn("Config option %s not set." % o)
return False
return True
else:
process = subprocess.Popen(['gcc', '-lzmq', '-lczmq'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outs, errs = process.communicate()
# Python 3
# try:
# outs, errs = process.communicate(timeout=15)
# except subprocess.TimeoutExpired:
# process.kill()
# outs, errs = process.communicate()
return (backwards.unicode2bytes('zmq') not in errs)
def new_comm_kwargs(cls,
name,
user=None,
password=None,
host=None,
virtual_host=None,
port=None,
exchange=None,
queue='',
**kwargs):
r"""Initialize communication with new connection.
Args:
name (str): Name of new connection.
user (str, optional): RabbitMQ server username. Defaults to config
option 'user' in section 'rmq' if it exists and 'guest' if it
does not.
password (str, optional): RabbitMQ server password. Defaults to
config option 'password' in section 'rmq' if it exists and
'guest' if it does not.
host (str, optional): RabbitMQ server host. Defaults to config option
'host' in section 'rmq' if it exists and 'localhost' if it
does not. If 'localhost', the output of socket.gethostname()
is used.
virtual_host (str, optional): RabbitMQ server virtual host. Defaults
to config option 'vhost' in section 'rmq' if it exists and '/'
if it does not.
port (str, optional): Port on host to use. Defaults to config option
'port' in section 'rmq' if it exists and '5672' if it does not.
exchange (str, optional): RabbitMQ exchange. Defaults to config
option 'namespace' in section 'rmq' if it exits and '' if it does
not.
queue (str, optional): Name of the queue that messages will be
send to or received from. If an empty string, the queue will
be a random string and exclusive to a receiving comm. Defaults
to ''.
**kwargs: Additional keywords arguments are returned as keyword
arguments for the new comm.
Returns:
tuple(tuple, dict): Arguments and keyword arguments for new comm.
"""
args = [name]
if 'address' not in kwargs:
if user is None:
user = cis_cfg.get('rmq', 'user', 'guest')
if password is None:
password = cis_cfg.get('rmq', 'password', 'guest')
if host is None:
host = cis_cfg.get('rmq', 'host', 'localhost')
# if host == 'localhost':
# host = socket.gethostname()
if virtual_host is None:
virtual_host = cis_cfg.get('rmq', 'vhost', '/')
if virtual_host == '/':
virtual_host = '%2f'
if port is None:
port = cis_cfg.get('rmq', 'port', '5672')
if exchange is None:
exchange = cis_cfg.get('rmq', 'namespace', '')
url = 'amqp://%s:%s@%s:%s/%s' % (user, password, host, port,
virtual_host)
kwargs['address'] = _rmq_param_sep.join([url, exchange, queue])
return args, kwargs
def debug_log(self): # pragma: debug
r"""Turn on debugging."""
self._old_loglevel = cis_cfg.get('debug', 'cis')
cis_cfg.set('debug', 'cis', 'DEBUG')
cfg_logging()
