def __init__ (self) :
if not hasattr (self, '_apitype') :
self._apitype = self._get_apitype ()
self._logger = rul.getLogger ('saga', self._apitype)
self._id = ru.generate_id (self._get_apitype () + '.', mode=ru.ID_SIMPLE)
def __init__(self, url, session=None, logger=None, opts=None, posix=True):
if logger: self.logger = logger
else: self.logger = rul.getLogger('saga', 'PTYShell')
if session: self.session = session
else: self.session = ss.Session(default=True)
if opts: self.options = opts
else: self.options = dict()
self.logger.debug("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.posix = posix # /bin/sh compatible?
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
self.pty_id = PTYShell._pty_id
PTYShell._pty_id += 1
self.cfg = self.session.get_config('saga.utils.pty')
# get prompt pattern from options, config, or use default
if 'prompt_pattern' in self.options:
self.prompt = self.options['prompt_pattern']
elif 'prompt_pattern' in self.cfg:
self.prompt = self.cfg['prompt_pattern'].get_value()
else:
self.prompt = DEFAULT_PROMPT
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs(self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.base):
pass
else:
raise se.NoSuccess("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory()
self.pty_info = self.factory.initialize(self.url,
self.session,
self.prompt,
self.logger,
posix=self.posix)
self.pty_shell = self.factory.run_shell(self.pty_info)
self._trace('init : %s' % self.pty_shell.command)
self.initialize()
def __init__(self):
if not hasattr(self, '_apitype'):
self._apitype = self._get_apitype()
self._logger = rul.getLogger('saga', self._apitype)
self._id = ru.generate_id(self._get_apitype() + '.', mode=ru.ID_SIMPLE)
def __init__ (self, adaptor_info, adaptor_options=[]) :
self._info = adaptor_info
self._opts = adaptor_options
self._name = adaptor_info['name']
self._schemas = adaptor_info['schemas']
self._lock = ru.RLock (self._name)
self._logger = rul.getLogger ('saga', self._name)
has_enabled = False
for option in self._opts :
if option['name'] == 'enabled' :
has_enabled = True
if not has_enabled :
# *every* adaptor needs an 'enabled' option!
self._opts.append ({
'category' : self._name,
'name' : 'enabled',
'type' : bool,
'default' : True,
'valid_options' : [True, False],
'documentation' : "Enable / disable loading of the adaptor",
'env_variable' : None
}
)
ruc.Configurable.__init__ (self, 'saga')
ruc.Configurable.config_options (self, self._name, self._opts)
def __init__ (self, url, session=None, logger=None, init=None, opts={}) :
if None != logger : self.logger = logger
else : self.logger = rul.getLogger ('saga', 'PTYShell')
self.logger.debug ("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.init = init # call after reconnect
self.opts = opts # options...
self.latency = 0.0 # set by factory
self.prompt = None
self.prompt_re = None
self.initialized = False
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs (self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir (self.base):
pass
else:
raise se.NoSuccess ("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory ()
self.pty_info = self.factory.initialize (url, session, self.logger)
self.pty_shell = self.factory.run_shell (self.pty_info)
self.initialize ()
def __init__(self, inp_file, rconfig):
"""Constructor.
Arguments:
inp_file - json input file with Pilot and NAMD related parameters as specified by user
"""
self.name = 'exec'
self.logger = rul.getLogger ('radical.repex', self.name)
# pilot parameters
self.resource = rconfig['target'].get('resource')
self.sandbox = rconfig['target'].get('sandbox')
self.user = rconfig['target'].get('username')
self.password = rconfig['target'].get('password')
self.project = rconfig['target'].get('project')
self.queue = rconfig['target'].get('queue')
self.cores = int(rconfig['target'].get('cores'))
self.runtime = int(rconfig['target'].get('runtime'))
self.dburl = rconfig['target'].get('mongo_url')
if self.dburl is None:
self.logger.info("Using default Mongo DB url" )
self.dburl = "mongodb://*****:*****@ds023438.mlab.com:23438/cdi-testing"
cleanup = rconfig['target'].get('cleanup','False')
if (cleanup == "True"):
self.cleanup = True
else:
self.cleanup = False
def __init__ (self, kernel_info) :
self._info = kernel_info
self._name = kernel_info['name']
self._subname = None
self._logger = rul.getLogger ('radical.enmd', self._name)
self._args = []
self._raw_args = []
self._pre_exec = None
self._post_exec = None
self._environment = None
self._executable = None
self._arguments = None
self._uses_mpi = None
self._cores = 1
self._upload_input_data = None
self._link_input_data = None
self._download_input_data = None
self._download_output_data = None
self._copy_input_data = None
self._copy_output_data = None
def __init__ (self, url, session=None, logger=None, opts=None, posix=True):
if logger : self.logger = logger
else : self.logger = rul.getLogger('saga', 'PTYShell')
if session: self.session = session
else : self.session = ss.Session(default=True)
if opts : self.options = opts
else : self.options = dict()
self.logger.debug ("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.posix = posix # /bin/sh compatible?
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
self.pty_id = PTYShell._pty_id
PTYShell._pty_id += 1
self.cfg = self.session.get_config('saga.utils.pty')
# get prompt pattern from options, config, or use default
if 'prompt_pattern' in self.options:
self.prompt = self.options['prompt_pattern']
elif 'prompt_pattern' in self.cfg:
self.prompt = self.cfg['prompt_pattern'].get_value ()
else:
self.prompt = DEFAULT_PROMPT
self.prompt_re = re.compile ("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info ("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs (self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir (self.base):
pass
else:
raise se.NoSuccess ("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory ()
self.pty_info = self.factory.initialize (self.url, self.session,
self.prompt, self.logger,
posix=self.posix)
self.pty_shell = self.factory.run_shell (self.pty_info)
self._trace ('init : %s' % self.pty_shell.command)
self.initialize ()
def __init__ (self, adaptor_info, adaptor_options=[]) :
self._info = adaptor_info
self._opts = adaptor_options
self._name = adaptor_info['name']
self._lock = ru.RLock (self._name)
self._logger = rul.getLogger ('radical.enmd', self._name)
def initialize (self, url, session=None, prompt=None, logger=None, posix=True) :
with self.rlock :
# make sure we have a valid url type
url = saga.Url (url)
if not prompt :
prompt = "^(.*[\$#%>\]])\s*$"
if not logger :
logger = rul.getLogger ('saga', 'PTYShellFactory')
# collect all information we have/need about the requested master
# connection
info = self._create_master_entry (url, session, prompt, logger, posix)
# we got master info - register the master, and create the instance!
type_s = str(info['shell_type'])
user_s = str(info['user'])
host_s = str(info['host_str'])
# Now, if we don't have that master, yet, we need to instantiate it
if not host_s in self.registry : self.registry[host_s] = {}
if not user_s in self.registry[host_s] : self.registry[host_s][user_s] = {}
if not type_s in self.registry[host_s][user_s] :
# new master: create an instance, and register it
m_cmd = info['scripts'][info['shell_type']]['master'] % info
logger.debug ("open master pty for [%s] [%s] %s: %s'" \
% (type_s, host_s, user_s, m_cmd))
info['pty'] = supp.PTYProcess (m_cmd, logger=logger)
if not info['pty'].alive () :
raise se.NoSuccess._log (logger, \
"Shell not connected to %s" % info['host_str'])
# authorization, prompt setup, etc. Initialize as shell if not
# explicitly marked as non-posix shell
self._initialize_pty (info['pty'], info)
# master was created - register it
self.registry[host_s][user_s][type_s] = info
else :
# we already have a master: make sure it is alive, and restart as
# needed
info = self.registry[host_s][user_s][type_s]
if not info['pty'].alive (recover=True) :
raise se.IncorrectState._log (logger, \
"Lost shell connection to %s" % info['host_str'])
return info
def initialize(self, url, session=None, prompt=None, logger=None):
with self.rlock:
# make sure we have a valid url type
url = saga.Url(url)
if not prompt:
prompt = "^(.*[\$#%>\]])\s*$"
if not logger:
logger = rul.getLogger('saga', 'PTYShellFactory')
# collect all information we have/need about the requested master
# connection
info = self._create_master_entry(url, session, prompt, logger)
# we got master info - register the master, and create the instance!
type_s = str(info['type'])
user_s = str(info['user'])
host_s = str(info['host_str'])
# Now, if we don't have that master, yet, we need to instantiate it
if not host_s in self.registry: self.registry[host_s] = {}
if not user_s in self.registry[host_s]:
self.registry[host_s][user_s] = {}
if not type_s in self.registry[host_s][user_s]:
# new master: create an instance, and register it
m_cmd = info['scripts'][info['type']]['master'] % info
logger.debug ("open master pty for [%s] [%s] %s: %s'" \
% (type_s, host_s, user_s, m_cmd))
info['pty'] = supp.PTYProcess(m_cmd, logger=logger)
if not info['pty'].alive():
raise se.NoSuccess._log (logger, \
"Shell not connected to %s" % info['host_str'])
# authorization, prompt setup, etc
self._initialize_pty(info['pty'], info, is_shell=True)
# master was created - register it
self.registry[host_s][user_s][type_s] = info
else:
# we already have a master: make sure it is alive, and restart as
# needed
info = self.registry[host_s][user_s][type_s]
if not info['pty'].alive(recover=True):
raise se.IncorrectState._log (logger, \
"Lost shell connection to %s" % info['host_str'])
return info
def __init__(self, session=None, *args, **kwargs):
self._session = session
if session:
self._logger = session._logger
else:
self._logger = rul.getLogger('saga', 'ContextList')
base_list = super(_ContextList, self)
base_list.__init__(*args, **kwargs)
def __init__ (self, session=None, *args, **kwargs) :
self._session = session
if session :
self._logger = session._logger
else :
self._logger = rul.getLogger ('saga', 'ContextList')
base_list = super (_ContextList, self)
base_list.__init__ (*args, **kwargs)
def __init__(self, url):
if url.scheme != 'redis':
raise BadParameter(
"scheme in url is not supported (%s != redis://...)" % url)
self.url = url
self.host = 'localhost'
self.port = 6379
self.db = 0
self.password = None
self.errors = 'strict'
if url.host: self.host = url.host
if url.port: self.port = url.port
if url.username: self.username = url.username
if url.password: self.password = url.password
# create redis client
t1 = time.time()
redis.Redis.__init__(self,
host=self.host,
port=self.port,
db=self.db,
password=self.password,
errors=self.errors)
t2 = time.time()
# add a logger
self.logger = rul.getLogger('saga', "redis-%s" % self.host)
# create a cache dict and attach to redis client instance. Cache
# lifetime is set to 10 times the redis-connect latency.
self.cache = redis_cache.Cache(logger=self.logger,
ttl=((t2 - t1) * 10))
# create a second client to manage the (blocking)
# pubsub communication for event notifications
self.r2 = redis.Redis(host=self.host,
port=self.port,
db=self.db,
password=self.password,
errors=self.errors)
# set up pubsub endpoint, and start a thread to monitor channels
self.callbacks = {}
self.pub = self.r2.pubsub()
self.pub.subscribe(MON)
# FIXME: create one pubsub channel per path (for paths which have
# callbacks registered)
self.monitor = redis_ns_monitor(self, self.pub)
self.monitor.start()
def __init__ (self, command, logger=None) :
"""
The class constructor, which runs (execvpe) command in a separately
forked process. The bew process will inherit the environment of the
application process.
:type command: string or list of strings
:param command: The given command is what is run as a child, and
fed/drained via pty pipes. If given as string, command is split into an
array of strings, using :func:`shlex.split`.
:type logger: :class:`radical.utils.logger.Logger` instance
:param logger: logger stream to send status messages to.
"""
self._debug = False
self.logger = logger
if not self.logger : self.logger = rul.getLogger ('saga', 'PTYProcess')
self.logger.debug ("PTYProcess init %s" % self)
if isinstance (command, basestring) :
command = shlex.split (command)
if not isinstance (command, list) :
raise se.BadParameter ("PTYProcess expects string or list command")
if len(command) < 1 :
raise se.BadParameter ("PTYProcess expects non-empty command")
self.rlock = ru.RLock ("pty process %s" % command)
self.command = command # list of strings too run()
self.cache = "" # data cache
self.tail = "" # tail of data data cache for error messages
self.child = None # the process as created by subprocess.Popen
self.ptyio = None # the process' io channel, from pty.fork()
self.exit_code = None # child died with code (may be revived)
self.exit_signal = None # child kill by signal (may be revived)
self.recover_max = 3 # TODO: make configure option. This does not
self.recover_attempts = 0 # apply for recovers triggered by gc_timeout!
try :
self.initialize ()
except Exception as e :
raise ptye.translate_exception (e, "pty or process creation failed")
def __init__ (self) :
# the default session picks up default contexts, from all context
# adaptors. To implemented, we have to do some legwork: get the engine,
# dig through the registered context adaptors, and ask each of them for
# default contexts.
self.contexts = _ContextList ()
self._logger = rul.getLogger ('saga', 'DefaultSession')
# FIXME: at the moment, the lease manager is owned by the session.
# Howevwer, the pty layer is the main user of the lease manager,
# and we thus keep the lease manager options in the pty subsection.
# So here we are, in the session, evaluating the pty config options...
config = saga.engine.engine.Engine ().get_config ('saga.utils.pty')
self._lease_manager = ru.LeaseManager (
max_pool_size = config['connection_pool_size'].get_value (),
max_pool_wait = config['connection_pool_wait'].get_value (),
max_obj_age = config['connection_pool_ttl'].get_value ()
)
_engine = saga.engine.engine.Engine ()
if not 'saga.Context' in _engine._adaptor_registry :
self._logger.warn ("no context adaptors found")
return
for schema in _engine._adaptor_registry['saga.Context'] :
for info in _engine._adaptor_registry['saga.Context'][schema] :
default_ctxs = []
try :
default_ctxs = info['adaptor_instance']._get_default_contexts ()
except se.SagaException as e :
self._logger.debug ("adaptor %s failed to provide default" \
"contexts: %s" % (info['adaptor_name'], e))
continue
for default_ctx in default_ctxs :
try :
self.contexts.append (ctx=default_ctx, session=self)
self._logger.debug ("default context [%-20s] : %s" \
% (info['adaptor_name'], default_ctx))
except se.SagaException as e :
self._logger.debug ("skip default context [%-20s] : %s : %s" \
% (info['adaptor_name'], default_ctx, e))
continue
def __init__(self, inp_file, rconfig):
"""Constructor.
Arguments:
inp_file - json input file with Pilot and NAMD related parameters as
specified by user
"""
PilotKernel.__init__(self, inp_file, rconfig)
self.name = 'exec-pattern-A-multiD'
self.logger = rul.getLogger ('radical.repex', self.name)
self.sd_shared_list = []
def __init__ (self, url) :
if url.scheme != 'redis' :
raise BadParameter ("scheme in url is not supported (%s != redis://...)" % url)
self.url = url
self.host = 'localhost'
self.port = 6379
self.db = 0
self.password = None
self.errors = 'strict'
if url.host : self.host = url.host
if url.port : self.port = url.port
if url.username : self.username = url.username
if url.password : self.password = url.password
# create redis client
t1 = time.time ()
redis.Redis.__init__ (self,
host = self.host,
port = self.port,
db = self.db,
password = self.password,
errors = self.errors)
t2 = time.time ()
# add a logger
self.logger = rul.getLogger ('saga', "redis-%s" % self.host)
# create a cache dict and attach to redis client instance. Cache
# lifetime is set to 10 times the redis-connect latency.
self.cache = redis_cache.Cache (logger=self.logger, ttl=((t2-t1)*10))
# create a second client to manage the (blocking)
# pubsub communication for event notifications
self.r2 = redis.Redis (host = self.host,
port = self.port,
db = self.db,
password = self.password,
errors = self.errors)
# set up pubsub endpoint, and start a thread to monitor channels
self.callbacks = {}
self.pub = self.r2.pubsub ()
self.pub.subscribe (MON)
# FIXME: create one pubsub channel per path (for paths which have
# callbacks registered)
self.monitor = redis_ns_monitor (self, self.pub)
self.monitor.start ()
def __init__ (self, url, session=None, logger=None, init=None, opts={}) :
# print 'new pty shell to %s' % url
if logger : self.logger = logger
else : self.logger = rul.getLogger ('saga', 'PTYShell')
if session : self.session = session
else : self.session = ss.Session (default=True)
self.logger.debug ("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.init = init # call after reconnect
self.opts = opts # options...
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
# get prompt pattern from config
self.cfg = self.session.get_config('saga.utils.pty')
if 'prompt_pattern' in self.cfg :
self.prompt = self.cfg['prompt_pattern'].get_value ()
self.prompt_re = re.compile ("^(.*?)%s" % self.prompt, re.DOTALL)
else :
self.prompt = "[\$#%>\]]\s*$"
self.prompt_re = re.compile ("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info ("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs (self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir (self.base):
pass
else:
raise se.NoSuccess ("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory ()
self.pty_info = self.factory.initialize (self.url, self.session, self.prompt, self.logger)
self.pty_shell = self.factory.run_shell (self.pty_info)
self.initialize ()
def __init__(self, url, session=None, logger=None, init=None, opts={}):
# print 'new pty shell to %s' % url
if logger: self.logger = logger
else: self.logger = rul.getLogger('saga', 'PTYShell')
if session: self.session = session
else: self.session = ss.Session(default=True)
self.logger.debug("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.init = init # call after reconnect
self.opts = opts # options...
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
# get prompt pattern from config
self.cfg = self.session.get_config('saga.utils.pty')
if 'prompt_pattern' in self.cfg:
self.prompt = self.cfg['prompt_pattern'].get_value()
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
else:
self.prompt = "[\$#%>\]]\s*$"
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs(self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.base):
pass
else:
raise se.NoSuccess("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory()
self.pty_info = self.factory.initialize(self.url, self.session,
self.prompt, self.logger)
self.pty_shell = self.factory.run_shell(self.pty_info)
self.initialize()
def __init__(self):
# the default session picks up default contexts, from all context
# adaptors. To implemented, we have to do some legwork: get the engine,
# dig through the registered context adaptors, and ask each of them for
# default contexts.
self.contexts = _ContextList()
self._logger = rul.getLogger('saga', 'DefaultSession')
# FIXME: at the moment, the lease manager is owned by the session.
# Howevwer, the pty layer is the main user of the lease manager,
# and we thus keep the lease manager options in the pty subsection.
# So here we are, in the session, evaluating the pty config options...
config = saga.engine.engine.Engine().get_config('saga.utils.pty')
self._lease_manager = ru.LeaseManager(
max_pool_size=config['connection_pool_size'].get_value(),
max_pool_wait=config['connection_pool_wait'].get_value(),
max_obj_age=config['connection_pool_ttl'].get_value())
_engine = saga.engine.engine.Engine()
if not 'saga.Context' in _engine._adaptor_registry:
self._logger.warn("no context adaptors found")
return
for schema in _engine._adaptor_registry['saga.Context']:
for info in _engine._adaptor_registry['saga.Context'][schema]:
default_ctxs = []
try:
default_ctxs = info[
'adaptor_instance']._get_default_contexts()
except se.SagaException as e:
self._logger.debug ("adaptor %s failed to provide default" \
"contexts: %s" % (info['adaptor_name'], e))
continue
for default_ctx in default_ctxs:
try:
self.contexts.append(ctx=default_ctx, session=self)
self._logger.debug ("default context [%-20s] : %s" \
% (info['adaptor_name'], default_ctx))
except se.SagaException as e:
self._logger.debug ("skip default context [%-20s] : %s : %s" \
% (info['adaptor_name'], default_ctx, e))
continue
def __init__(self):
# Engine manages cpis from adaptors
self._adaptor_registry = {}
# set the configuration options for this object
ruc.Configurable.__init__(self, 'saga')
ruc.Configurable.config_options(self, 'saga.engine', _config_options)
self._cfg = self.get_config('saga.engine')
# Initialize the logging, and log version (this is a singleton!)
self._logger = rul.getLogger('saga', 'Engine')
# load adaptors
self._load_adaptors()
def __init__ (self, max_pool_size=MAX_POOL_SIZE, max_pool_wait=MAX_POOL_WAIT, max_obj_age=MAX_OBJ_AGE) :
"""
Make sure the object dict is initialized, exactly once.
"""
import radical.utils.logger as rul
self._log = rul.getLogger('radical.utils')
# self._log.setLevel ('DEBUG')
self._log.debug ('lm new manager')
self._pools = dict()
self._max_pool_size = max_pool_size
self._max_pool_wait = max_pool_wait
self._max_obj_age = max_obj_age
def __init__(self, inp_file, work_dir_local):
"""Constructor.
Arguments:
inp_file - package input file with Pilot and NAMD related parameters as specified by user
work_dir_local - directory from which main simulation script was invoked
"""
self.shared_urls = []
self.shared_files = []
self.name = 'nk-patternB-tex'
self.logger = rul.getLogger ('radical.repex', self.name)
NamdKernelTex.__init__(self, inp_file, work_dir_local)
def __init__(self, command, logger=None):
"""
The class constructor, which runs (execvpe) command in a separately
forked process. The bew process will inherit the environment of the
application process.
:type command: string or list of strings
:param command: The given command is what is run as a child, and
fed/drained via pty pipes. If given as string, command is split into an
array of strings, using :func:`shlex.split`.
:type logger: :class:`radical.utils.logger.Logger` instance
:param logger: logger stream to send status messages to.
"""
self.logger = logger
if not self.logger: self.logger = rul.getLogger('saga', 'PTYProcess')
self.logger.debug("PTYProcess init %s" % self)
if isinstance(command, basestring):
command = shlex.split(command)
if not isinstance(command, list):
raise se.BadParameter("PTYProcess expects string or list command")
if len(command) < 1:
raise se.BadParameter("PTYProcess expects non-empty command")
self.rlock = ru.RLock("pty process %s" % command)
self.command = command # list of strings too run()
self.cache = "" # data cache
self.tail = "" # tail of data data cache for error messages
self.child = None # the process as created by subprocess.Popen
self.ptyio = None # the process' io channel, from pty.fork()
self.exit_code = None # child died with code (may be revived)
self.exit_signal = None # child kill by signal (may be revived)
self.recover_max = 3 # TODO: make configure option. This does not
self.recover_attempts = 0 # apply for recovers triggered by gc_timeout!
try:
self.initialize()
except Exception as e:
raise ptye.translate_exception(e, "pty or process creation failed")
def __init__(self, namespace):
"""
namespace: name of module (plugins are expected in namespace/plugins/)
"""
# import here to avoid circular imports
import radical.utils.logger as logger
self._namespace = namespace
self._logger = logger.getLogger("radical")
self._registry = _PluginRegistry() # singleton
self._plugins = self._registry.retrieve(self._namespace)
# load adaptors if registry didn't have any registered, yet
if not self._plugins:
self._load_plugins()
self._registry.register(self._namespace, self._plugins)
def __init__(self, namespace):
"""
namespace: name of module (plugins are expected in namespace/plugins/)
"""
# import here to avoid circular imports
import radical.utils.logger as logger
self._namespace = namespace
self._logger = logger.getLogger('radical')
self._registry = _PluginRegistry() # singleton
self._plugins = self._registry.retrieve(self._namespace)
# load adaptors if registry didn't have any registered, yet
if not self._plugins:
self._load_plugins()
self._registry.register(self._namespace, self._plugins)
def __init__(self,
max_pool_size=MAX_POOL_SIZE,
max_pool_wait=MAX_POOL_WAIT,
max_obj_age=MAX_OBJ_AGE):
"""
Make sure the object dict is initialized, exactly once.
"""
import radical.utils.logger as rul
self._log = rul.getLogger('radical.utils')
self._log.setLevel('DEBUG')
self._log.debug('lm new manager')
self._pools = dict()
self._max_pool_size = max_pool_size
self._max_pool_wait = max_pool_wait
self._max_obj_age = max_obj_age
def __init__(self):
# Engine manages cpis from adaptors
self._adaptor_registry = {}
# set the configuration options for this object
ruc.Configurable.__init__ (self, 'saga')
ruc.Configurable.config_options (self, 'saga.engine', _config_options)
self._cfg = self.get_config('saga.engine')
# Initialize the logging, and log version (this is a singleton!)
self._logger = rul.getLogger ('saga', 'Engine')
# load adaptors
self._load_adaptors ()
def __init__(self):
# the default session picks up default contexts, from all context
# adaptors. To implemented, we have to do some legwork: get the engine,
# dig through the registered context adaptors, and ask each of them for
# default contexts.
self.contexts = _ContextList()
self._lease_manager = ru.LeaseManager()
self._logger = rul.getLogger('saga', 'DefaultSession')
_engine = saga.engine.engine.Engine()
if not 'saga.Context' in _engine._adaptor_registry:
self._logger.warn("no context adaptors found")
return
for schema in _engine._adaptor_registry['saga.Context']:
for info in _engine._adaptor_registry['saga.Context'][schema]:
default_ctxs = []
try:
default_ctxs = info[
'adaptor_instance']._get_default_contexts()
except se.SagaException as e:
self._logger.debug ("adaptor %s failed to provide default" \
"contexts: %s" % (info['adaptor_name'], e))
continue
for default_ctx in default_ctxs:
try:
self.contexts.append(ctx=default_ctx, session=self)
self._logger.debug ("default context [%-20s] : %s" \
% (info['adaptor_name'], default_ctx))
except se.SagaException as e:
self._logger.debug ("skip default context [%-20s] : %s : %s" \
% (info['adaptor_name'], default_ctx, e))
continue
def __init__ (self) :
# the default session picks up default contexts, from all context
# adaptors. To implemented, we have to do some legwork: get the engine,
# dig through the registered context adaptors, and ask each of them for
# default contexts.
self.contexts = _ContextList ()
self._logger = rul.getLogger ('saga', 'DefaultSession')
_engine = saga.engine.engine.Engine ()
if not 'saga.Context' in _engine._adaptor_registry :
self._logger.warn ("no context adaptors found")
return
for schema in _engine._adaptor_registry['saga.Context'] :
for info in _engine._adaptor_registry['saga.Context'][schema] :
default_ctxs = []
try :
default_ctxs = info['adaptor_instance']._get_default_contexts ()
except se.SagaException as e :
self._logger.debug ("adaptor %s failed to provide default" \
"contexts: %s" % (info['adaptor_name'], e))
continue
for default_ctx in default_ctxs :
try :
self.contexts.append (ctx=default_ctx, session=self)
self._logger.debug ("default context [%-20s] : %s" \
% (info['adaptor_name'], default_ctx))
except se.SagaException as e :
self._logger.debug ("skip default context [%-20s] : %s : %s" \
% (info['adaptor_name'], default_ctx, e))
continue
def __init__ (self, api, adaptor) :
self._session = None
self._adaptor = adaptor
self._cpi_cname = self.__class__.__name__
self._logger = rul.getLogger ('saga', self._cpi_cname)
# The API object must obviously keep an adaptor instance. If we also
# keep an API instance ref in this adaptor base, we create a ref cycle
# which will annoy (i.e. disable) garbage collection. We thus use weak
# references to break that cycle. The inheriting classes MUST use
# get_api() to obtain the API reference.
if api :
self._api = weakref.ref (api)
else :
self._api = None
# by default, we assume that no bulk optimizations are supported by the
# adaptor class. Any adaptor class supporting bulks ops must overwrite
# the ``_container`` attribute (via
# ``self._set_container(container=None)``, and have it point to the
# class which implements the respective ``container_*`` methods.
self._container = None
def __init__(self, api, adaptor):
self._session = None
self._adaptor = adaptor
self._cpi_cname = self.__class__.__name__
self._logger = rul.getLogger('saga', self._cpi_cname)
# The API object must obviously keep an adaptor instance. If we also
# keep an API instance ref in this adaptor base, we create a ref cycle
# which will annoy (i.e. disable) garbage collection. We thus use weak
# references to break that cycle. The inheriting classes MUST use
# get_api() to obtain the API reference.
# if api :
# self._api = weakref.ref (api)
# else :
# self._api = None
self._api = weakref.ref(api)
# by default, we assume that no bulk optimizations are supported by the
# adaptor class. Any adaptor class supporting bulks ops must overwrite
# the ``_container`` attribute (via
# ``self._set_container(container=None)``, and have it point to the
# class which implements the respective ``container_*`` methods.
self._container = None
def __init__ (self) :
self.logger = rul.getLogger ('saga', 'PTYShellFactory')
self.registry = {}
self.rlock = ru.RLock ('pty shell factory')
self._cp_slaves = dict()
def __init__(self):
if not hasattr(self, '_apitype'):
self._apitype = self._get_apitype()
self._logger = rul.getLogger('saga', self._apitype)
def __init__(self, inp_file, rconfig, work_dir_local):
"""Constructor.
Arguments:
inp_file - package input file with Pilot and NAMD related parameters as
specified by user
work_dir_local - directory from which main simulation script was invoked
"""
self.dims = 3
self.resource = rconfig['target']['resource']
if 'number_of_cycles' in inp_file['remd.input']:
self.nr_cycles = int(inp_file['remd.input']['number_of_cycles'])
else:
self.nr_cycles = None
self.input_folder = inp_file['remd.input']['input_folder']
self.inp_basename = inp_file['remd.input']['input_file_basename']
self.amber_coordinates_path = inp_file['remd.input']['amber_coordinates_folder']
if 'same_coordinates' in inp_file['remd.input']:
coors = inp_file['remd.input']['same_coordinates']
if coors == "True":
self.same_coordinates = True
else:
self.same_coordinates = False
else:
self.same_coordinates = True
if 'download_mdinfo' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdinfo'] == 'True':
self.down_mdinfo = True
else:
self.down_mdinfo = False
else:
self.down_mdinfo = True
if 'download_mdout' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdout'] == 'True':
self.down_mdout = True
else:
self.down_mdout = False
else:
self.down_mdout = True
self.amber_parameters = inp_file['remd.input']['amber_parameters']
self.amber_input = inp_file['remd.input']['amber_input']
#-----------------------------------------------------------------------
if 'exchange_off' in inp_file['remd.input']:
if inp_file['remd.input']['exchange_off'] == "True":
self.exchange_off = True
else:
self.exchange_off = False
else:
self.exchange_off = False
#-----------------------------------------------------------------------
if 'replica_mpi' in inp_file['remd.input']:
if inp_file['remd.input']['replica_mpi'] == "True":
self.md_replica_mpi = True
else:
self.md_replica_mpi = False
else:
self.md_replica_mpi= False
if 'replica_cores' in inp_file['remd.input']:
self.md_replica_cores = int(inp_file['remd.input']['replica_cores'])
else:
self.md_replica_cores = 1
self.cycle_steps = int(inp_file['remd.input']['steps_per_cycle'])
self.work_dir_local = work_dir_local
self.us_template = inp_file['remd.input']['us_template']
self.current_cycle = -1
# hardcoded for now
self.replicas_d1 = int(inp_file['dim.input']\
['umbrella_sampling_1']["number_of_replicas"])
self.replicas_d2 = int(inp_file['dim.input']\
['temperature_2']["number_of_replicas"])
self.replicas_d3 = int(inp_file['dim.input']\
['umbrella_sampling_3']["number_of_replicas"])
#-----------------------------------------------------------------------
# hardcoding dimension names
self.d1 = 'umbrella_sampling'
self.d2 = 'temperature'
self.d3 = 'umbrella_sampling'
self.replicas = self.replicas_d1 * self.replicas_d2 * self.replicas_d3
self.restraints_files = []
for k in range(self.replicas):
self.restraints_files.append(self.us_template + "." + str(k) )
self.us_start_param_d1 = float(inp_file['dim.input']\
['umbrella_sampling_1']['us_start_param'])
self.us_end_param_d1 = float(inp_file['dim.input']\
['umbrella_sampling_1']['us_end_param'])
self.us_start_param_d3 = float(inp_file['dim.input']\
['umbrella_sampling_3']['us_start_param'])
self.us_end_param_d3 = float(inp_file['dim.input']\
['umbrella_sampling_3']['us_end_param'])
self.min_temp = float(inp_file['dim.input']\
['temperature_2']['min_temperature'])
self.max_temp = float(inp_file['dim.input']\
['temperature_2']['max_temperature'])
#-----------------------------------------------------------------------
self.name = 'ak-patternB-3d-TUU'
self.logger = rul.getLogger ('radical.repex', self.name)
self.pre_exec = KERNELS[self.resource]["kernels"]\
["amber"]["pre_execution"]
try:
self.amber_path = inp_file['remd.input']['amber_path']
except:
self.logger.info("Using default Amber path for: {0}"\
.format(rconfig['target']['resource']) )
try:
self.amber_path = KERNELS[self.resource]["kernels"]\
["amber"]["executable"]
except:
self.logger.info("Amber path for localhost is not defined!")
self.shared_urls = []
self.shared_files = []
self.all_temp_list = []
self.all_rstr_list_d1 = []
self.all_rstr_list_d3 = []
self.d1_id_matrix = []
self.d2_id_matrix = []
self.d3_id_matrix = []
self.temp_matrix = []
self.us_d1_matrix = []
self.us_d3_matrix = []
def __init__(self, inp_file, rconfig, work_dir_local):
"""Constructor.
Arguments:
inp_file - package input file with Pilot and NAMD related parameters as
specified by user
work_dir_local - directory from which main simulation script was invoked
"""
self.name = 'amber-tremd-patternA'
self.ex_name = 'temperature'
self.logger = rul.getLogger ('radical.repex', self.name)
if 'number_of_cycles' in inp_file['remd.input']:
self.nr_cycles = int(inp_file['remd.input']['number_of_cycles'])
else:
self.nr_cycles = None
if 'replica_mpi' in inp_file['remd.input']:
mpi = inp_file['remd.input']['replica_mpi']
if mpi == "True":
self.replica_mpi = True
else:
self.replica_mpi = False
else:
self.replica_mpi = False
if 'exchange_mpi' in inp_file['remd.input']:
mpi = inp_file['remd.input']['exchange_mpi']
if mpi == "True":
self.exchange_mpi = True
else:
self.exchange_mpi = False
else:
self.exchange_mpi = False
if 'replica_cores' in inp_file['remd.input']:
self.replica_cores = int(inp_file['remd.input']['replica_cores'])
else:
self.replica_cores = 1
self.resource = rconfig['target']['resource']
self.cores = int(rconfig['target']['cores'])
self.pre_exec = KERNELS[self.resource]["kernels"]["amber"]["pre_execution"]
if 'download_mdinfo' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdinfo'] == 'True':
self.down_mdinfo = True
else:
self.down_mdinfo = False
else:
self.down_mdinfo = True
if 'download_mdout' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdout'] == 'True':
self.down_mdout = True
else:
self.down_mdout = False
else:
self.down_mdout = True
if self.resource == 'local.localhost':
if 'amber_path' in inp_file['remd.input']:
self.amber_path = inp_file['remd.input']['amber_path']
else:
self.logger.info("Amber path for {0} is not defined".format( rconfig['target']['resource'] ) )
sys.exit(1)
else:
if 'amber_path' in inp_file['remd.input']:
self.amber_path = inp_file['remd.input']['amber_path']
else:
self.logger.info("Using default Amber path for: {0}".format( rconfig['target']['resource'] ) )
try:
self.amber_path = KERNELS[self.resource]["kernels"]["amber"]["executable"]
except:
self.logger.info("Amber path for {0} is not defined".format( rconfig['target']['resource'] ) )
self.input_folder = inp_file['remd.input']['input_folder']
self.amber_coordinates = inp_file['remd.input']['amber_coordinates']
self.amber_parameters = inp_file['remd.input']['amber_parameters']
self.amber_input = inp_file['remd.input']['amber_input']
self.input_folder = inp_file['remd.input']['input_folder']
self.inp_basename = inp_file['remd.input']['input_file_basename']
self.replicas = int(inp_file['remd.input']['number_of_replicas'])
self.min_temp = float(inp_file['remd.input']['min_temperature'])
self.max_temp = float(inp_file['remd.input']['max_temperature'])
self.cycle_steps = int(inp_file['remd.input']['steps_per_cycle'])
self.work_dir_local = work_dir_local
self.current_cycle = -1
self.shared_urls = []
self.shared_files = []
self.all_temp_list = []
def __init__(self, inp_file, rconfig, work_dir_local):
"""Constructor.
Arguments:
inp_file - package input file with Pilot and NAMD related parameters as
specified by user
work_dir_local - directory from which main simulation script was invoked
"""
self.resource = rconfig['target']['resource']
self.inp_basename = inp_file['remd.input']['input_file_basename']
self.input_folder = inp_file['remd.input']['input_folder']
self.replicas = int(inp_file['remd.input']['number_of_replicas'])
self.cores = int(rconfig['target']['cores'])
self.us_template = inp_file['remd.input']['us_template']
self.init_temperature = float(inp_file['remd.input']['init_temperature'])
self.cycle_steps = int(inp_file['remd.input']['steps_per_cycle'])
self.work_dir_local = work_dir_local
try:
self.nr_cycles = int(inp_file['remd.input']['number_of_cycles'])
except:
self.nr_cycles = None
if 'replica_mpi' in inp_file['remd.input']:
mpi = inp_file['remd.input']['replica_mpi']
if mpi == "True":
self.replica_mpi = True
else:
self.replica_mpi = False
else:
self.replica_mpi = False
if 'exchange_mpi' in inp_file['remd.input']:
mpi = inp_file['remd.input']['exchange_mpi']
if mpi == "True":
self.exchange_mpi = True
else:
self.exchange_mpi = False
else:
self.exchange_mpi = False
if 'download_mdinfo' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdinfo'] == 'True':
self.down_mdinfo = True
else:
self.down_mdinfo = False
else:
self.down_mdinfo = True
if 'download_mdout' in inp_file['remd.input']:
if inp_file['remd.input']['download_mdout'] == 'True':
self.down_mdout = True
else:
self.down_mdout = False
else:
self.down_mdout = True
try:
self.replica_cores = inp_file['remd.input']['replica_cores']
except:
self.replica_cores = 1
self.restraints_files = []
for k in range(self.replicas):
self.restraints_files.append(self.us_template + "." + str(k) )
self.pre_exec = KERNELS[self.resource]["kernels"]["amber"]["pre_execution"]
try:
self.amber_path = inp_file['remd.input']['amber_path']
except:
print "Using default Amber path for %s" % rconfig['target']['resource']
try:
self.amber_path = KERNELS[self.resource]["kernels"]["amber"]["executable"]
except:
print "Amber path for localhost is not defined..."
self.amber_coordinates_path = inp_file['remd.input']['amber_coordinates_folder']
if 'same_coordinates' in inp_file['remd.input']:
coors = inp_file['remd.input']['same_coordinates']
if coors == "True":
self.same_coordinates = True
else:
self.same_coordinates = False
else:
self.same_coordinates = True
self.amber_parameters = inp_file['remd.input']['amber_parameters']
self.amber_input = inp_file['remd.input']['amber_input']
self.input_folder = inp_file['remd.input']['input_folder']
self.us_start_param = float(inp_file['remd.input']['us_start_param'])
self.us_end_param = float(inp_file['remd.input']['us_end_param'])
self.current_cycle = -1
self.name = 'ak-us-patternB'
self.ex_name = 'umbrella_sampling'
self.logger = rul.getLogger ('radical.repex', self.name)
self.shared_urls = []
self.shared_files = []
def __init__(self):
self.logger = rul.getLogger('saga', 'PTYShellFactory')
self.registry = {}
self.rlock = ru.RLock('pty shell factory')
def __init__ (self) :
if not hasattr (self, '_apitype') :
self._apitype = self._get_apitype ()
self._logger = rul.getLogger ('saga', self._apitype)
