def start_forwarder_backend():
if parse_bool(config.get_conf('beah-backend').get('DEFAULT', 'DEVEL')):
print_this = log_this(lambda s: log.debug(s), log_on=True)
make_class_verbose(ForwarderBackend, print_this)
make_class_verbose(_RemoteBackend, print_this)
backend = ForwarderBackend()
# Start a default TCP client:
start_backend(backend)
return backend
def start_forwarder_backend():
if parse_bool(config.get_conf('beah-backend').get('DEFAULT', 'DEVEL')):
print_this = log_this(lambda s: log.debug(s), log_on=True)
make_class_verbose(ForwarderBackend, print_this)
make_class_verbose(_RemoteBackend, print_this)
backend = ForwarderBackend()
# Start a default TCP client:
start_backend(backend)
return backend
def start_watchdog_backend(conf):
'''Starts a watchdog backend with specified configuration.'''
log = logging.getLogger('backend')
if parse_bool(conf.get('DEFAULT', 'DEVEL')):
print_this = log_this(log.debug, log_on=True)
make_class_verbose(WatchdogBackend, print_this)
make_class_verbose(Task, print_this)
handlers = dict(DEFAULT_HANDLERS)
load_plugins(WATCHDOG_ENTRYPOINT, handlers)
backend = WatchdogBackend(conf=conf, log=log, handlers=handlers)
query_interval = int(conf.get('DEFAULT', 'QUERY_WATCHDOG'))
if query_interval > 0:
watchdogs_request = LoopingCall(backend.query_watchdogs)
watchdogs_request.start(query_interval, now=False)
#pylint: disable=E1101
reactor.addSystemEventTrigger('before', 'shutdown', watchdogs_request.stop)
# Start a default TCP client:
backend.initialize()
start_backend(backend)
def start_watchdog_backend(conf):
'''Starts a watchdog backend with specified configuration.'''
log = logging.getLogger('backend')
if parse_bool(conf.get('DEFAULT', 'DEVEL')):
print_this = log_this(log.debug, log_on=True)
make_class_verbose(WatchdogBackend, print_this)
make_class_verbose(Task, print_this)
handlers = dict(DEFAULT_HANDLERS)
load_plugins(WATCHDOG_ENTRYPOINT, handlers)
backend = WatchdogBackend(conf=conf, log=log, handlers=handlers)
query_interval = int(conf.get('DEFAULT', 'QUERY_WATCHDOG'))
if query_interval > 0:
watchdogs_request = LoopingCall(backend.query_watchdogs)
watchdogs_request.start(query_interval, now=False)
#pylint: disable=E1101
reactor.addSystemEventTrigger('before', 'shutdown',
watchdogs_request.stop)
# Start a default TCP client:
backend.initialize()
start_backend(backend)
def main():
################################################################################
# EXECUTE:
################################################################################
global conf, runtime, uploader, results
conf = conf_main({}, sys.argv[1:])
name = conf['name']
var_path = os.path.join(conf['root'], VAR_PATH, name)
runtime = runtimes.ShelveRuntime(os.path.join(var_path, "runtime"))
log = logging.getLogger('beah_fakelc')
twisted_logging(log, level=logging.WARNING)
# FIXME: redirect to console or syslog?
misc.make_log_handler(log, os.path.join(conf['root'], LOG_PATH),
"%s.log" % name)
log.setLevel(misc.str2log_level(conf.get('LOG', 'warning')))
if conf.get('DEVEL', False):
print_this = log_this.log_this(lambda s: log.debug(s), log_on=True)
# make the classes verbose:
misc.make_class_verbose(LCRecipes, print_this)
misc.make_class_verbose(LCRecipeTasks, print_this)
misc.make_class_verbose(LCHandler, print_this)
lc = LCHandler()
lc.XMLRPC_TIMEOUT = safe_int(conf['timeout'], 0)
s = server.Site(lc, None, 60 * 60 * 12)
reactor.listenTCP(conf['port'], s, interface='')
# To support testing in IPv6 and mixed IPv4/IPv6 environments,
# we attempt to listen on the specified (defaults to IPv6)
# interface as well.
try:
reactor.listenTCP(conf['port'], s, interface=conf['interface'])
except CannotListenError:
pass
reactor.addSystemEventTrigger("before", "shutdown", close, log)
upload_path = os.path.join(var_path, "fakelc-uploads")
results = Results(var_path, upload_path)
uploader = Uploader(upload_path)
reactor.run()
def main():
################################################################################
# EXECUTE:
################################################################################
global conf, runtime, uploader, results
conf = conf_main({}, sys.argv[1:])
name = conf['name']
var_path = os.path.join(conf['root'], VAR_PATH, name)
runtime = runtimes.ShelveRuntime(os.path.join(var_path, "runtime"))
log = logging.getLogger('beah_fakelc')
twisted_logging(log, level=logging.WARNING)
# FIXME: redirect to console or syslog?
misc.make_log_handler(log, os.path.join(conf['root'], LOG_PATH), "%s.log" % name)
log.setLevel(misc.str2log_level(conf.get('LOG', 'warning')))
if conf.get('DEVEL', False):
print_this = log_this.log_this(lambda s: log.debug(s), log_on=True)
# make the classes verbose:
misc.make_class_verbose(LCRecipes, print_this)
misc.make_class_verbose(LCRecipeTasks, print_this)
misc.make_class_verbose(LCHandler, print_this)
lc = LCHandler()
lc.XMLRPC_TIMEOUT = safe_int(conf['timeout'], 0)
s = server.Site(lc, None, 60*60*12)
reactor.listenTCP(conf['port'], s, interface='')
# To support testing in IPv6 and mixed IPv4/IPv6 environments,
# we attempt to listen on the specified (defaults to IPv6)
# interface as well.
try:
reactor.listenTCP(conf['port'], s, interface=conf['interface'])
except CannotListenError:
pass
reactor.addSystemEventTrigger("before", "shutdown", close, log)
upload_path = os.path.join(var_path, "fakelc-uploads")
results = Results(var_path, upload_path)
uploader = Uploader(upload_path)
reactor.run()
def start_server(conf=None, backend_host=None, backend_port=None,
backend_adaptor=BackendAdaptor_JSON,
task_host=None, task_port=None,
task_adaptor=TaskAdaptor_JSON, spawn=None):
# CONFIG:
if not conf:
config.beah_conf()
conf = config.get_conf('beah')
# LOGGING:
twisted_logging(log, level=logging.WARNING)
log.setLevel(str2log_level(conf.get('CONTROLLER', 'LOG')))
# Create a directory for runtime
# FIXME: should try to create a temp path if following fails:
ensuredir(conf.get('CONTROLLER', 'VAR_ROOT'))
# Create a directory for logging and check permissions
lp = conf.get('CONTROLLER', 'LOG_PATH')
make_log_handler(log, lp, conf.get('CONTROLLER', 'LOG_FILE_NAME'),
syslog=True, console=conf.get('CONTROLLER', 'CONSOLE_LOG', False))
if parse_bool(config.get_conf('beah').get('CONTROLLER', 'DEVEL')):
print_this = log_this(log.debug, True)
make_class_verbose(Controller, print_this)
make_class_verbose(MasterTask, print_this)
# RUN:
backend_host = backend_host or conf.get('BACKEND', 'INTERFACE')
backend_port = backend_port or conf.get('BACKEND', 'PORT')
task_host = task_host or conf.get('TASK', 'INTERFACE')
task_port = task_port or int(conf.get('TASK', 'PORT'))
if os.name == 'posix':
if backend_port != '':
backend_port = int(backend_port)
backend_socket = conf.get('BACKEND', 'SOCKET')
if task_port != '':
task_port = int(task_port)
task_socket = conf.get('TASK', 'SOCKET')
else:
backend_port = int(backend_port)
backend_socket = ''
task_port = int(task_port)
task_socket = ''
runtime = runtimes.ShelveRuntime(conf.get('CONTROLLER', 'RUNTIME_FILE_NAME'))
runtime.vars = runtimes.TypeDict(runtime, 'vars')
runtime.tasks = runtimes.TypeDict(runtime, 'tasks')
controller = Controller(spawn or Spawn(task_host, task_port, socket=task_socket), runtime=runtime)
def on_killed():
if not controller.backends:
reactor.stop()
return
reactor.callLater(2, reactor.stop)
controller.on_killed = on_killed
log.info("################################")
log.info("# Starting a Controller... #")
log.info("################################")
backend_listener = BackendListener(controller, backend_adaptor)
if backend_port != '':
if backend_host == 'localhost':
listening = listen_loopback_tcp(backend_port, backend_listener)
elif backend_host:
listening = reactor.listenTCP(backend_port, backend_listener, interface=backend_host)
else:
listening = listen_all_tcp(backend_port,
backend_listener,
ipv6_disabled=parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')))
log.info("Controller: BackendListener listening on %s port %s",
listening.getHost().host, listening.getHost().port)
if backend_socket:
if os.path.exists(backend_socket):
# clean-up after e.g. system crash:
log.warning("Controller: BackendListener cleaning %s", backend_socket)
os.remove(backend_socket)
log.info("Controller: BackendListener listening on %s", backend_socket)
reactor.listenUNIX(backend_socket, backend_listener)
task_listener = TaskListener(controller, task_adaptor)
if task_port != '':
if task_host == 'localhost':
listening = listen_loopback_tcp(task_port, task_listener)
elif task_host:
listening = reactor.listenTCP(task_port, task_listener, interface=task_host)
else:
listening = listen_all_tcp(task_port,
task_listener,
ipv6_disabled=parse_bool(conf.get('DEFAULT', 'IPV6_DISABLED')))
log.info("Controller: TaskListener listening on %s port %s",
listening.getHost().host, listening.getHost().port)
if task_socket:
if os.path.exists(task_socket):
# clean-up after e.g. system crash:
log.warning("Controller: TaskListener cleaning %s", task_socket)
os.remove(task_socket)
log.info("Controller: TaskListener listening on %s", task_socket)
reactor.listenUNIX(task_socket, task_listener)
return controller
raise TypeError('Each arg must be a callable.', arg)
self.args.append(arg)
return self
def __call__(self, data):
for arg in self.args:
arg(data)
################################################################################
# Test:
################################################################################
if __name__ == '__main__':
import traceback
from beah.misc import log_this
import pprint
pretty_log = log_this.log_this(pprint.pprint)
print "1..1"
print "# BASIC USAGE:"
try:
ow = ObjectWriter()
ow.proc_obj([])
ow.proc_obj({'a':[('H3110', 1.0, None)]})
ow = Pprinter()
ow.proc_obj({'a':[('H3110', 1.0, None)]})
except:
raise
print "# ADVANCED USAGE:"
a = Deserializer()
a.redir(Tee(
def start_server(conf=None,
backend_host=None,
backend_port=None,
backend_adaptor=BackendAdaptor_JSON,
task_host=None,
task_port=None,
task_adaptor=TaskAdaptor_JSON,
spawn=None):
# CONFIG:
if not conf:
config.beah_conf()
conf = config.get_conf('beah')
# LOGGING:
twisted_logging(log, level=logging.WARNING)
log.setLevel(str2log_level(conf.get('CONTROLLER', 'LOG')))
# Create a directory for runtime
# FIXME: should try to create a temp path if following fails:
ensuredir(conf.get('CONTROLLER', 'VAR_ROOT'))
# Create a directory for logging and check permissions
lp = conf.get('CONTROLLER', 'LOG_PATH')
make_log_handler(log,
lp,
conf.get('CONTROLLER', 'LOG_FILE_NAME'),
syslog=True,
console=conf.get('CONTROLLER', 'CONSOLE_LOG', False))
if parse_bool(config.get_conf('beah').get('CONTROLLER', 'DEVEL')):
print_this = log_this(log.debug, True)
make_class_verbose(Controller, print_this)
make_class_verbose(MasterTask, print_this)
# RUN:
backend_host = backend_host or conf.get('BACKEND', 'INTERFACE')
backend_port = backend_port or conf.get('BACKEND', 'PORT')
task_host = task_host or conf.get('TASK', 'INTERFACE')
task_port = task_port or int(conf.get('TASK', 'PORT'))
if os.name == 'posix':
if backend_port != '':
backend_port = int(backend_port)
backend_socket = conf.get('BACKEND', 'SOCKET')
if task_port != '':
task_port = int(task_port)
task_socket = conf.get('TASK', 'SOCKET')
else:
backend_port = int(backend_port)
backend_socket = ''
task_port = int(task_port)
task_socket = ''
runtime = runtimes.ShelveRuntime(
conf.get('CONTROLLER', 'RUNTIME_FILE_NAME'))
runtime.vars = runtimes.TypeDict(runtime, 'vars')
runtime.tasks = runtimes.TypeDict(runtime, 'tasks')
controller = Controller(spawn
or Spawn(task_host, task_port, socket=task_socket),
runtime=runtime)
def on_killed():
if not controller.backends:
reactor.stop()
return
reactor.callLater(2, reactor.stop)
controller.on_killed = on_killed
log.info("################################")
log.info("# Starting a Controller... #")
log.info("################################")
backend_listener = BackendListener(controller, backend_adaptor)
if backend_port != '':
if backend_host == 'localhost':
listening = listen_loopback_tcp(backend_port, backend_listener)
elif backend_host:
listening = reactor.listenTCP(backend_port,
backend_listener,
interface=backend_host)
else:
listening = listen_all_tcp(backend_port,
backend_listener,
ipv6_disabled=parse_bool(
conf.get('DEFAULT',
'IPV6_DISABLED')))
log.info("Controller: BackendListener listening on %s port %s",
listening.getHost().host,
listening.getHost().port)
if backend_socket:
if os.path.exists(backend_socket):
# clean-up after e.g. system crash:
log.warning("Controller: BackendListener cleaning %s",
backend_socket)
os.remove(backend_socket)
log.info("Controller: BackendListener listening on %s", backend_socket)
reactor.listenUNIX(backend_socket, backend_listener)
task_listener = TaskListener(controller, task_adaptor)
if task_port != '':
if task_host == 'localhost':
listening = listen_loopback_tcp(task_port, task_listener)
elif task_host:
listening = reactor.listenTCP(task_port,
task_listener,
interface=task_host)
else:
listening = listen_all_tcp(task_port,
task_listener,
ipv6_disabled=parse_bool(
conf.get('DEFAULT',
'IPV6_DISABLED')))
log.info("Controller: TaskListener listening on %s port %s",
listening.getHost().host,
listening.getHost().port)
if task_socket:
if os.path.exists(task_socket):
# clean-up after e.g. system crash:
log.warning("Controller: TaskListener cleaning %s", task_socket)
os.remove(task_socket)
log.info("Controller: TaskListener listening on %s", task_socket)
reactor.listenUNIX(task_socket, task_listener)
return controller
return self
def __call__(self, data):
for arg in self.args:
arg(data)
################################################################################
# Test:
################################################################################
if __name__ == '__main__':
import traceback
from beah.misc import log_this
import pprint
pretty_log = log_this.log_this(pprint.pprint)
print "1..1"
print "# BASIC USAGE:"
try:
ow = ObjectWriter()
ow.proc_obj([])
ow.proc_obj({'a': [('H3110', 1.0, None)]})
ow = Pprinter()
ow.proc_obj({'a': [('H3110', 1.0, None)]})
except:
raise
print "# ADVANCED USAGE:"
a = Deserializer()
a.redir(
VERBOSE = False
if VERBOSE:
DelayedCall.debug = True
def printf_w_timestamp(s):
#ts = time.strftime("%Y%m%d-%H%M%S")
ts = time.time()
print("%.2f: %s" % (ts, s))
sys.stdout.flush()
printf = printf_w_timestamp
print_this = log_this(printf)
class TestRepeaters(unittest.TestCase):
def testRepeatAlways(self):
for i in range(7):
assert repeatingproxy.repeatAlways(i)
def testRepeatTimes(self):
repeat6Times = repeatingproxy.repeatTimes(6)
try:
for i in range(6):
assert repeat6Times(i)
assert not repeat6Times(7)
finally:
