def start_backend(backend, host=None, port=None,
adaptor=ControllerAdaptor_Backend_JSON,
byef=None):
conf = config.get_conf('beah-backend')
host = host or conf.get('DEFAULT', 'INTERFACE')
port = port or conf.get('DEFAULT', 'PORT')
if os.name == 'posix':
socket = conf.get('DEFAULT', 'SOCKET')
# 0. check SOCKET_OPT (socket given on command line)
if parse_bool(conf.get('DEFAULT', 'SOCKET_OPT')) and socket != '':
port = ''
# 1. check INTERFACE - if not empty nor localhost: must use TCP
if not localhost_(host):
socket = ''
# 2. check PORT_OPT (port given on command line)
if parse_bool(conf.get('DEFAULT', 'PORT_OPT')) and port != '':
socket = ''
else:
socket = ''
backend_factory = BackendFactory(backend, adaptor, byef)
if socket != '':
return reactor.connectUNIX(socket, backend_factory)
elif port and host:
return reactor.connectTCP(host, int(port), backend_factory)
elif port:
if not parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')):
return connect_loopback(int(port),
backend_factory)
else:
return connect_loopback(int(port),
backend_factory,
ipv6_disabled=True)
else:
raise EnvironmentError('Either socket or port must be configured.')
def testIntegration(self):
# CONFIG:
config.beah_conf(args=[])
conf = config.get_conf('beah')
srv = start_server(conf=conf)
log = logging.getLogger('backend')
class FakeTask(object):
origin = {'signature':'FakeTask'}
task_id = 'no-id'
def proc_cmd(self, cmd):
log.debug("FakeTask.proc_cmd(%r)", cmd)
class FakeBackend(object):
def proc_evt(self, evt, **kwargs):
log.debug("FakeBackend.proc_evt(%r, **%r)", evt, kwargs)
t = FakeTask()
reactor.callLater(2, srv.proc_evt, t, event.variable_set('say_hi', 'Hello World!'))
#reactor.callLater(2.1, srv.proc_evt, t, event.variable_get('say_hi'))
reactor.callLater(2.2, srv.proc_evt, t, event.variable_get('say_hi', dest='test.loop'))
b = FakeBackend()
reactor.callLater(3, srv.proc_cmd, b, command.kill())
forwarder.main(args=[])
def testIntegration(self):
# CONFIG:
config.beah_conf(args=[])
conf = config.get_conf('beah')
srv = start_server(conf=conf)
log = logging.getLogger('backend')
class FakeTask(object):
origin = {'signature': 'FakeTask'}
task_id = 'no-id'
def proc_cmd(self, cmd):
log.debug("FakeTask.proc_cmd(%r)", cmd)
class FakeBackend(object):
def proc_evt(self, evt, **kwargs):
log.debug("FakeBackend.proc_evt(%r, **%r)", evt, kwargs)
t = FakeTask()
reactor.callLater(2, srv.proc_evt, t,
event.variable_set('say_hi', 'Hello World!'))
#reactor.callLater(2.1, srv.proc_evt, t, event.variable_get('say_hi'))
reactor.callLater(2.2, srv.proc_evt, t,
event.variable_get('say_hi', dest='test.loop'))
b = FakeBackend()
reactor.callLater(3, srv.proc_cmd, b, command.kill())
forwarder.main(args=[])
def main():
from beah.core import debug
config.beah_conf()
conf = config.get_conf('beah')
debug.setup(os.getenv("BEAH_SRV_DEBUGGER"), conf.get('DEFAULT', 'NAME'))
print main_srv.__doc__
main_srv(conf=conf)
debug.runcall(reactor.run)
def start_forwarder_backend():
if parse_bool(config.get_conf('beah-backend').get('DEFAULT', 'DEVEL')):
print_this = log_this(lambda s: log.debug(s), log_on=True)
make_class_verbose(ForwarderBackend, print_this)
make_class_verbose(_RemoteBackend, print_this)
backend = ForwarderBackend()
# Start a default TCP client:
start_backend(backend)
return backend
def start_forwarder_backend():
if parse_bool(config.get_conf('beah-backend').get('DEFAULT', 'DEVEL')):
print_this = log_this(lambda s: log.debug(s), log_on=True)
make_class_verbose(ForwarderBackend, print_this)
make_class_verbose(_RemoteBackend, print_this)
backend = ForwarderBackend()
# Start a default TCP client:
start_backend(backend)
return backend
def _test_conf():
"""Self test."""
_tst_eq(config._Config.parse_conf_name('NAME'), ('DEFAULT', 'NAME'))
_tst_eq(config._Config.parse_conf_name('SEC.NAME'), ('SEC', 'NAME'))
fd, fn = tempfile.mkstemp()
try:
os.close(fd)
c = config._BeahConfig(
'test', '_BEAH_CONF', '_beah.conf', config.beah_defaults(),
dict(ROOT='/mnt/testarea',
LOG='False',
DEVEL='True',
_BEAH_CONF=fn))
_tst_eq(c._get_conf_file(fn), fn)
_tst_eq(c._get_conf_file(), '')
_tst_eq(c._get_conf_file('empty-beah.conf.tmp'), '')
cfg = c.conf
cfg.set('BACKEND', 'INTERFACE', "127.0.0.1")
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'DEVEL')), True)
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'LOG')), False)
_tst_eq(cfg.get('BACKEND', 'INTERFACE'), "127.0.0.1")
_tst_eq(int(cfg.get('BACKEND', 'PORT')), 12432)
_tst_eq(cfg.get('DEFAULT', 'ROOT'), "/mnt/testarea")
cfg.set('DEFAULT', 'LOG', 'True')
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'DEVEL')), True)
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'LOG')), True)
_tst_eq(cfg.get('BACKEND', 'INTERFACE'), "127.0.0.1")
_tst_eq(int(cfg.get('BACKEND', 'PORT')), 12432)
_tst_eq(cfg.get('DEFAULT', 'ROOT'), "/mnt/testarea")
config._Config._remove('test')
finally:
os.remove(fn)
try:
c = config._Config('test2', None, 'empty-missing-no.conf',
dict(GREETING="Hello %(NAME)s!"),
dict(NAME="World"))
try:
config._Config._remove('test2')
except:
pass
raise RuntimeError("this should have failed with exception")
except:
pass
c = config._Config('test3', None, None, dict(GREETING="Hello %(NAME)s!"),
dict(NAME="World"))
_tst_eq(c.conf.get('DEFAULT', 'GREETING'), 'Hello World!')
_tst_eq(
config.get_conf('test3').get('DEFAULT', 'GREETING'), 'Hello World!')
config._Config._remove('test3')
def configure():
'''\
Returns a watchdog backend configuration.
This uses command line options, environment and config.file as sources in
that order.
'''
config.backend_conf(env_var='BEAH_WATCHDOG_CONF', filename='beah_watchdog.conf',
defaults=defaults(), overrides=config.backend_opts(option_adder=watchdog_opts))
return config.get_conf('beah-backend')
def _test_conf():
"""Self test."""
_tst_eq(config._Config.parse_conf_name('NAME'), ('DEFAULT', 'NAME'))
_tst_eq(config._Config.parse_conf_name('SEC.NAME'), ('SEC', 'NAME'))
fd, fn = tempfile.mkstemp()
try:
os.close(fd)
c = config._BeahConfig('test', '_BEAH_CONF', '_beah.conf',
config.beah_defaults(),
dict(ROOT='/mnt/testarea', LOG='False', DEVEL='True', _BEAH_CONF=fn))
_tst_eq(c._get_conf_file(fn), fn)
_tst_eq(c._get_conf_file(), '')
_tst_eq(c._get_conf_file('empty-beah.conf.tmp'), '')
cfg = c.conf
cfg.set('BACKEND', 'INTERFACE', "127.0.0.1")
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'DEVEL')), True)
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'LOG')), False)
_tst_eq(cfg.get('BACKEND', 'INTERFACE'), "127.0.0.1")
_tst_eq(int(cfg.get('BACKEND', 'PORT')), 12432)
_tst_eq(cfg.get('DEFAULT', 'ROOT'), "/mnt/testarea")
cfg.set('DEFAULT', 'LOG', 'True')
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'DEVEL')), True)
_tst_eq(test_parse_bool(cfg.get('DEFAULT', 'LOG')), True)
_tst_eq(cfg.get('BACKEND', 'INTERFACE'), "127.0.0.1")
_tst_eq(int(cfg.get('BACKEND', 'PORT')), 12432)
_tst_eq(cfg.get('DEFAULT', 'ROOT'), "/mnt/testarea")
config._Config._remove('test')
finally:
os.remove(fn)
try:
c = config._Config('test2', None, 'empty-missing-no.conf',
dict(GREETING="Hello %(NAME)s!"), dict(NAME="World"))
try:
config._Config._remove('test2')
except:
pass
raise RuntimeError("this should have failed with exception")
except:
pass
c = config._Config('test3', None, None,
dict(GREETING="Hello %(NAME)s!"),
dict(NAME="World"))
_tst_eq(c.conf.get('DEFAULT', 'GREETING'), 'Hello World!')
_tst_eq(config.get_conf('test3').get('DEFAULT', 'GREETING'), 'Hello World!')
config._Config._remove('test3')
def configure():
'''\
Returns a watchdog backend configuration.
This uses command line options, environment and config.file as sources in
that order.
'''
config.backend_conf(
env_var='BEAH_WATCHDOG_CONF',
filename='beah_watchdog.conf',
defaults=defaults(),
overrides=config.backend_opts(option_adder=watchdog_opts))
return config.get_conf('beah-backend')
def log_handler(log_file_name=None):
conf = config.get_conf('beah-backend')
if not log_file_name:
if conf.has_option('DEFAULT', 'LOG_FILE'):
log_file_name = conf.get('DEFAULT', 'LOG_FILE')
else:
log_file_name = conf.get('DEFAULT', 'NAME') + '.log'
lp = conf.get('DEFAULT', 'LOG_PATH') or "/var/log"
log = logging.getLogger('backend')
twisted_logging(log, level=logging.WARNING)
make_log_handler(log, lp, log_file_name, syslog=True,
console=parse_bool(conf.get('DEFAULT', 'CONSOLE_LOG')))
log.setLevel(str2log_level(conf.get('DEFAULT', 'LOG')))
return log
def test_defaults(self):
"""Check the default options are present"""
beakerlc.configure(args=[])
cfg = config._get_config('beah-backend')
conf = config.get_conf('beah-backend')
#cfg.print_()
#conf.write(sys.stdout)
assert conf.has_option('DEFAULT', 'NAME')
assert conf.has_option('DEFAULT', 'LAB_CONTROLLER')
assert conf.has_option('DEFAULT', 'RECIPEID')
assert conf.has_option('DEFAULT', 'HOSTNAME')
assert conf.has_option('DEFAULT', 'INTERFACE')
assert conf.has_option('DEFAULT', 'PORT')
assert conf.has_option('DEFAULT', 'LOG')
assert conf.has_option('DEFAULT', 'DEVEL')
assert conf.has_option('DEFAULT', 'VAR_ROOT')
assert conf.has_option('DEFAULT', 'LOG_PATH')
assert conf.has_option('DEFAULT', 'RUNTIME_FILE_NAME')
assert conf.has_option('DEFAULT', 'DIGEST')
assert conf.has_option('DEFAULT', 'RPC_TIMEOUT')
config._Config._remove('beah-backend')
def test_defaults(self):
"""Check the default options are present"""
beakerlc.configure(args=[])
cfg = config._get_config('beah-backend')
conf = config.get_conf('beah-backend')
#cfg.print_()
#conf.write(sys.stdout)
assert conf.has_option('DEFAULT', 'NAME')
assert conf.has_option('DEFAULT', 'LAB_CONTROLLER')
assert conf.has_option('DEFAULT', 'RECIPEID')
assert conf.has_option('DEFAULT', 'HOSTNAME')
assert conf.has_option('DEFAULT', 'INTERFACE')
assert conf.has_option('DEFAULT', 'PORT')
assert conf.has_option('DEFAULT', 'LOG')
assert conf.has_option('DEFAULT', 'DEVEL')
assert conf.has_option('DEFAULT', 'VAR_ROOT')
assert conf.has_option('DEFAULT', 'LOG_PATH')
assert conf.has_option('DEFAULT', 'RUNTIME_FILE_NAME')
assert conf.has_option('DEFAULT', 'DIGEST')
assert conf.has_option('DEFAULT', 'RPC_TIMEOUT')
config._Config._remove('beah-backend')
def spawn(controller, backend, task_info, env, args):
task_env = dict(env)
# 1. set env.variables
# BEAH_THOST - host name
# BEAH_TPORT - port
# BEAH_TSOCKET - socket
# BEAH_TID - id of task - used to introduce itself when opening socket
task_id = task_info['id']
conf = config.get_conf('beah')
env_file = os.path.join(conf.get('TASK', 'VAR_ROOT'),
ENV_PATHNAME_TEMPLATE % task_id)
dict_update(
task_env,
CALLED_BY_BEAH="1",
BEAH_THOST=str(host),
BEAH_TPORT=str(port),
BEAH_TSOCKET=str(socket),
BEAH_TID=str(task_id),
BEAH_ROOT=conf.get('TASK', 'ROOT'),
BEAH_ENV=env_file,
)
ll = conf.get('TASK', 'LOG')
task_env.setdefault('BEAH_TASK_LOG', ll)
task_env.setdefault('BEAH_TASK_CONSOLE',
conf.get('TASK', 'CONSOLE_LOG', 'False'))
task_env.setdefault('TERM', 'dumb')
val = os.getenv('PYTHONPATH')
if val:
task_env['PYTHONPATH'] = val
jsonenv.export_env(env_file, task_env)
# 2. spawn a task
protocol = (proto or TaskStdoutProtocol)(task_id)
protocol.controller = controller
log.debug('spawn: Environment: %r.', task_env)
reactor.spawnProcess(protocol,
task_info['file'],
args=[task_info['file']] + (args or []),
env=task_env)
# FIXME: send an answer to backend(?)
return protocol.task_protocol
def spawn(controller, backend, task_info, env, args):
task_env = dict(env)
# 1. set env.variables
# BEAH_THOST - host name
# BEAH_TPORT - port
# BEAH_TSOCKET - socket
# BEAH_TID - id of task - used to introduce itself when opening socket
task_id = task_info['id']
conf = config.get_conf('beah')
env_file = os.path.join(conf.get('TASK', 'VAR_ROOT'),
ENV_PATHNAME_TEMPLATE % task_id)
dict_update(task_env,
CALLED_BY_BEAH="1",
BEAH_THOST=str(host),
BEAH_TPORT=str(port),
BEAH_TSOCKET=str(socket),
BEAH_TID=str(task_id),
BEAH_ROOT=conf.get('TASK', 'ROOT'),
BEAH_ENV=env_file,
)
ll = conf.get('TASK', 'LOG')
task_env.setdefault('BEAH_TASK_LOG', ll)
task_env.setdefault('BEAH_TASK_CONSOLE', conf.get('TASK', 'CONSOLE_LOG', 'False'))
task_env.setdefault('TERM', 'dumb')
val = os.getenv('PYTHONPATH')
if val:
task_env['PYTHONPATH'] = val
jsonenv.export_env(env_file, task_env)
# 2. spawn a task
protocol = (proto or TaskStdoutProtocol)(task_id)
protocol.controller = controller
log.debug('spawn: Environment: %r.', task_env)
reactor.spawnProcess(protocol, task_info['file'],
args=[task_info['file']]+(args or []), env=task_env)
# FIXME: send an answer to backend(?)
return protocol.task_protocol
def __init__(self):
self.__remotes = {}
ExtBackend.__init__(self)
self.def_port = config.get_conf('beah-backend').get('DEFAULT', 'PORT')
def __init__(self):
self.__remotes = {}
ExtBackend.__init__(self)
self.def_port = config.get_conf('beah-backend').get('DEFAULT', 'PORT')
def start_server(conf=None, backend_host=None, backend_port=None,
backend_adaptor=BackendAdaptor_JSON,
task_host=None, task_port=None,
task_adaptor=TaskAdaptor_JSON, spawn=None):
# CONFIG:
if not conf:
config.beah_conf()
conf = config.get_conf('beah')
# LOGGING:
twisted_logging(log, level=logging.WARNING)
log.setLevel(str2log_level(conf.get('CONTROLLER', 'LOG')))
# Create a directory for runtime
# FIXME: should try to create a temp path if following fails:
ensuredir(conf.get('CONTROLLER', 'VAR_ROOT'))
# Create a directory for logging and check permissions
lp = conf.get('CONTROLLER', 'LOG_PATH')
make_log_handler(log, lp, conf.get('CONTROLLER', 'LOG_FILE_NAME'),
syslog=True, console=conf.get('CONTROLLER', 'CONSOLE_LOG', False))
if parse_bool(config.get_conf('beah').get('CONTROLLER', 'DEVEL')):
print_this = log_this(log.debug, True)
make_class_verbose(Controller, print_this)
make_class_verbose(MasterTask, print_this)
# RUN:
backend_host = backend_host or conf.get('BACKEND', 'INTERFACE')
backend_port = backend_port or conf.get('BACKEND', 'PORT')
task_host = task_host or conf.get('TASK', 'INTERFACE')
task_port = task_port or int(conf.get('TASK', 'PORT'))
if os.name == 'posix':
if backend_port != '':
backend_port = int(backend_port)
backend_socket = conf.get('BACKEND', 'SOCKET')
if task_port != '':
task_port = int(task_port)
task_socket = conf.get('TASK', 'SOCKET')
else:
backend_port = int(backend_port)
backend_socket = ''
task_port = int(task_port)
task_socket = ''
runtime = runtimes.ShelveRuntime(conf.get('CONTROLLER', 'RUNTIME_FILE_NAME'))
runtime.vars = runtimes.TypeDict(runtime, 'vars')
runtime.tasks = runtimes.TypeDict(runtime, 'tasks')
controller = Controller(spawn or Spawn(task_host, task_port, socket=task_socket), runtime=runtime)
def on_killed():
if not controller.backends:
reactor.stop()
return
reactor.callLater(2, reactor.stop)
controller.on_killed = on_killed
log.info("################################")
log.info("# Starting a Controller... #")
log.info("################################")
backend_listener = BackendListener(controller, backend_adaptor)
if backend_port != '':
if backend_host == 'localhost':
listening = listen_loopback_tcp(backend_port, backend_listener)
elif backend_host:
listening = reactor.listenTCP(backend_port, backend_listener, interface=backend_host)
else:
listening = listen_all_tcp(backend_port,
backend_listener,
ipv6_disabled=parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')))
log.info("Controller: BackendListener listening on %s port %s",
listening.getHost().host, listening.getHost().port)
if backend_socket:
if os.path.exists(backend_socket):
# clean-up after e.g. system crash:
log.warning("Controller: BackendListener cleaning %s", backend_socket)
os.remove(backend_socket)
log.info("Controller: BackendListener listening on %s", backend_socket)
reactor.listenUNIX(backend_socket, backend_listener)
task_listener = TaskListener(controller, task_adaptor)
if task_port != '':
if task_host == 'localhost':
listening = listen_loopback_tcp(task_port, task_listener)
elif task_host:
listening = reactor.listenTCP(task_port, task_listener, interface=task_host)
else:
listening = listen_all_tcp(task_port,
task_listener,
ipv6_disabled=parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')))
log.info("Controller: TaskListener listening on %s port %s",
listening.getHost().host, listening.getHost().port)
if task_socket:
if os.path.exists(task_socket):
# clean-up after e.g. system crash:
log.warning("Controller: TaskListener cleaning %s", task_socket)
os.remove(task_socket)
log.info("Controller: TaskListener listening on %s", task_socket)
reactor.listenUNIX(task_socket, task_listener)
return controller
def start_server(conf=None,
backend_host=None,
backend_port=None,
backend_adaptor=BackendAdaptor_JSON,
task_host=None,
task_port=None,
task_adaptor=TaskAdaptor_JSON,
spawn=None):
# CONFIG:
if not conf:
config.beah_conf()
conf = config.get_conf('beah')
# LOGGING:
twisted_logging(log, level=logging.WARNING)
log.setLevel(str2log_level(conf.get('CONTROLLER', 'LOG')))
# Create a directory for runtime
# FIXME: should try to create a temp path if following fails:
ensuredir(conf.get('CONTROLLER', 'VAR_ROOT'))
# Create a directory for logging and check permissions
lp = conf.get('CONTROLLER', 'LOG_PATH')
make_log_handler(log,
lp,
conf.get('CONTROLLER', 'LOG_FILE_NAME'),
syslog=True,
console=conf.get('CONTROLLER', 'CONSOLE_LOG', False))
if parse_bool(config.get_conf('beah').get('CONTROLLER', 'DEVEL')):
print_this = log_this(log.debug, True)
make_class_verbose(Controller, print_this)
make_class_verbose(MasterTask, print_this)
# RUN:
backend_host = backend_host or conf.get('BACKEND', 'INTERFACE')
backend_port = backend_port or conf.get('BACKEND', 'PORT')
task_host = task_host or conf.get('TASK', 'INTERFACE')
task_port = task_port or int(conf.get('TASK', 'PORT'))
if os.name == 'posix':
if backend_port != '':
backend_port = int(backend_port)
backend_socket = conf.get('BACKEND', 'SOCKET')
if task_port != '':
task_port = int(task_port)
task_socket = conf.get('TASK', 'SOCKET')
else:
backend_port = int(backend_port)
backend_socket = ''
task_port = int(task_port)
task_socket = ''
runtime = runtimes.ShelveRuntime(
conf.get('CONTROLLER', 'RUNTIME_FILE_NAME'))
runtime.vars = runtimes.TypeDict(runtime, 'vars')
runtime.tasks = runtimes.TypeDict(runtime, 'tasks')
controller = Controller(spawn
or Spawn(task_host, task_port, socket=task_socket),
runtime=runtime)
def on_killed():
if not controller.backends:
reactor.stop()
return
reactor.callLater(2, reactor.stop)
controller.on_killed = on_killed
log.info("################################")
log.info("# Starting a Controller... #")
log.info("################################")
backend_listener = BackendListener(controller, backend_adaptor)
if backend_port != '':
if backend_host == 'localhost':
listening = listen_loopback_tcp(backend_port, backend_listener)
elif backend_host:
listening = reactor.listenTCP(backend_port,
backend_listener,
interface=backend_host)
else:
listening = listen_all_tcp(backend_port,
backend_listener,
ipv6_disabled=parse_bool(
conf.get('DEFAULT',
'IPV6_DISABLED')))
log.info("Controller: BackendListener listening on %s port %s",
listening.getHost().host,
listening.getHost().port)
if backend_socket:
if os.path.exists(backend_socket):
# clean-up after e.g. system crash:
log.warning("Controller: BackendListener cleaning %s",
backend_socket)
os.remove(backend_socket)
log.info("Controller: BackendListener listening on %s", backend_socket)
reactor.listenUNIX(backend_socket, backend_listener)
task_listener = TaskListener(controller, task_adaptor)
if task_port != '':
if task_host == 'localhost':
listening = listen_loopback_tcp(task_port, task_listener)
elif task_host:
listening = reactor.listenTCP(task_port,
task_listener,
interface=task_host)
else:
listening = listen_all_tcp(task_port,
task_listener,
ipv6_disabled=parse_bool(
conf.get('DEFAULT',
'IPV6_DISABLED')))
log.info("Controller: TaskListener listening on %s port %s",
listening.getHost().host,
listening.getHost().port)
if task_socket:
if os.path.exists(task_socket):
# clean-up after e.g. system crash:
log.warning("Controller: TaskListener cleaning %s", task_socket)
os.remove(task_socket)
log.info("Controller: TaskListener listening on %s", task_socket)
reactor.listenUNIX(task_socket, task_listener)
return controller
