def signal_handler(sig, frm): # Supress buffering of re-issued messages self.handler._emit, self.handler.emit = self.handler.emit, lambda *a, **k: None for msg in list(self.buffer): log.fatal(self.handler.format(msg)) self.handler.emit = self.handler._emit
def main():
import argparse
parser = argparse.ArgumentParser(
description='Start the IRC helper bot.')
parser.add_argument('-e', '--relay-enable',
action='append', metavar='relay', default=list(),
help='Enable only the specified relays, can be specified multiple times.')
parser.add_argument('-d', '--relay-disable',
action='append', metavar='relay', default=list(),
help='Explicitly disable specified relays,'
' can be specified multiple times. Overrides --relay-enable.')
parser.add_argument('-c', '--config',
action='append', metavar='path', default=list(),
help='Configuration files to process.'
' Can be specified more than once.'
' Values from the latter ones override values in the former.'
' Available CLI options override the values in any config.')
parser.add_argument('-n', '--dry-run', action='store_true',
help='Connect to IRC, but do not communicate there,'
' dumping lines-to-be-sent to the log instead.')
parser.add_argument('--fatal-errors', action='store_true',
help='Do not try to ignore entry_point'
' init errors, bailing out with traceback instead.')
parser.add_argument('--debug',
action='store_true', help='Verbose operation mode.')
parser.add_argument('--debug-memleaks', action='store_true',
help='Import guppy and enable its manhole to debug memleaks (requires guppy module).')
parser.add_argument('--noise',
action='store_true', help='Even more verbose mode than --debug.')
optz = parser.parse_args()
## Read configuration files
cfg = lya.AttrDict.from_yaml('{}.yaml'.format(splitext(realpath(__file__))[0]))
for k in optz.config: cfg.update_yaml(k)
## CLI overrides
if optz.dry_run: cfg.debug.dry_run = optz.dry_run
## Logging
import logging
logging.NOISE = logging.DEBUG - 1
logging.addLevelName(logging.NOISE, 'NOISE')
try: from twisted.python.logger._stdlib import fromStdlibLogLevelMapping
except ImportError: pass # newer twisted versions only
else: fromStdlibLogLevelMapping[logging.NOISE] = logging.NOISE
if optz.noise: lvl = logging.NOISE
elif optz.debug: lvl = logging.DEBUG
else: lvl = logging.WARNING
lya.configure_logging(cfg.logging, lvl)
log.PythonLoggingObserver().start()
for lvl in 'noise', 'debug', 'info', ('warning', 'warn'), 'error', ('critical', 'fatal'):
lvl, func = lvl if isinstance(lvl, tuple) else (lvl, lvl)
assert not hasattr(log, lvl)
setattr(log, func, ft.partial( log.msg,
logLevel=logging.getLevelName(lvl.upper()) ))
# Manholes
if optz.debug_memleaks:
import guppy
from guppy.heapy import Remote
Remote.on()
## Fake "xattr" module, if requested
if cfg.core.xattr_emulation:
import shelve
xattr_db = shelve.open(cfg.core.xattr_emulation, 'c')
class xattr_path(object):
def __init__(self, base):
assert isinstance(base, str)
self.base = base
def key(self, k): return '{}\0{}'.format(self.base, k)
def __setitem__(self, k, v): xattr_db[self.key(k)] = v
def __getitem__(self, k): return xattr_db[self.key(k)]
def __del__(self): xattr_db.sync()
class xattr_module(object): xattr = xattr_path
sys.modules['xattr'] = xattr_module
## Actual init
# Merge entry points configuration with CLI opts
conf = ep_config( cfg,
[ dict(ep='relay_defaults'),
dict( ep='modules',
enabled=optz.relay_enable, disabled=optz.relay_disable ) ] )
(conf_base, conf), (conf_def_base, conf_def) =\
op.itemgetter('modules', 'relay_defaults')(conf)
for subconf in conf.viewvalues(): subconf.rebase(conf_base)
relays, channels, routes = (
dict( (name, subconf) for name,subconf in conf.viewitems()
if name[0] != '_' and subconf.get('type') == subtype )
for subtype in ['relay', 'channel', 'route'] )
# Init interface
interface = routing.BCInterface(
irc_enc=cfg.core.encoding,
chan_prefix=cfg.core.channel_prefix,
max_line_length=cfg.core.max_line_length,
dry_run=cfg.debug.dry_run )
# Find out which relay entry_points are actually used
route_mods = set(it.chain.from_iterable(
it.chain.from_iterable(
(mod if isinstance(mod, list) else [mod])
for mod in ((route.get(k) or list()) for k in ['src', 'dst', 'pipe']) )
for route in routes.viewvalues() ))
for name in list(route_mods):
try:
name_ep = relays[name].name
if name == name_ep: continue
except KeyError: pass
else:
route_mods.add(name_ep)
route_mods.remove(name)
# Init relays
relays_obj = dict()
for ep in get_relay_list():
if ep.name[0] == '_':
log.debug( 'Skipping entry_point with name'
' prefixed by underscore: {}'.format(ep.name) )
continue
if ep.name not in route_mods:
log.debug(( 'Skipping loading relay entry_point {}'
' because its not used in any of the routes' ).format(ep.name))
continue
ep_relays = list( (name, subconf)
for name, subconf in relays.viewitems()
if subconf.get('name', name) == ep.name )
if not ep_relays: ep_relays = [(ep.name, conf_base.clone())]
for name, subconf in ep_relays:
try: relay_defaults = conf_def[ep.name]
except KeyError: pass
else:
subconf.rebase(relay_defaults)
subconf.rebase(conf_def_base)
if subconf.get('enabled', True):
log.debug('Loading relay: {} ({})'.format(name, ep.name))
try:
obj = ep.load().relay(subconf, interface=interface)
if not obj: raise AssertionError('Empty object')
except Exception as err:
if optz.fatal_errors: raise
log.error('Failed to load/init relay {}: {} {}'.format(ep.name, type(err), err))
obj, subconf.enabled = None, False
if obj and subconf.get('enabled', True): relays_obj[name] = obj
else:
log.debug(( 'Entry point object {!r} (name:'
' {}) was disabled after init' ).format(obj, ep.name) )
for name in set(relays).difference(relays_obj):
log.debug(( 'Unused relay configuration - {}: no such'
' entry point - {}' ).format(name, relays[name].get('name', name)))
if not relays_obj:
log.fatal('No relay objects were properly enabled/loaded, bailing out')
sys.exit(1)
log.debug('Enabled relays: {}'.format(relays_obj))
# Relays-client interface
interface.update(relays_obj, channels, routes)
# Server
if cfg.core.connection.server.endpoint:
password = cfg.core.connection.get('password')
if not password:
from hashlib import sha1
password = cfg.core.connection.password =\
sha1(open('/dev/urandom', 'rb').read(120/8)).hexdigest()
factory = irc.BCServerFactory(
cfg.core.connection.server,
*(chan.get('name', name) for name,chan in channels.viewitems()),
**{cfg.core.connection.nickname: password} )
endpoints\
.serverFromString(reactor, cfg.core.connection.server.endpoint)\
.listen(factory)
# Client with proper endpoints + reconnection
# See: http://twistedmatrix.com/trac/ticket/4472 + 4700 + 4735
ep = endpoints.clientFromString(reactor, cfg.core.connection.endpoint)
irc.BCClientFactory(cfg.core, interface, ep).connect()
log.debug('Starting event loop')
reactor.run()
def update(self, relays, channels, routes):
def resolve(route, k, fork=False, lvl=0):
if k not in route: route[k] = list()
elif isinstance(route[k], types.StringTypes): route[k] = [route[k]]
modules = list()
for v in route[k]:
if v not in routes: modules.append(v)
else:
for subroute in routes[v]:
if fork is None:
resolve(subroute, k, lvl=lvl+1)
modules.extend(subroute[k])
else:
fork = route.clone()
fork.pipe = (list(fork.pipe) + subroute.pipe)\
if fork is True else (subroute.pipe + list(fork.pipe))
resolve(subroute, k, fork=True, lvl=lvl+1)
fork[k] = subroute[k]
routes[route.name].append(fork)
route[k] = modules
for name, route in routes.viewitems():
if not route.get('pipe'): route.pipe = list()
route.name = name
routes[name] = [route]
for k, fork in ('pipe', None), ('src', True), ('dst', False):
for name, route_set in routes.items():
for route in route_set:
if k == 'pipe':
for v in route.pipe or list():
if v in channels:
log.fatal( 'Channels are not allowed'
' in route.pipe sections (route: {}, channel: {})'.format(name, v) )
sys.exit(1)
resolve(route, k, fork=fork)
pipes, pipes_chk = dict(), set()
pipes_valid = set(relays).union(channels)
for route in it.chain.from_iterable(routes.viewvalues()):
if not route.src or not route.dst: continue
for src, dst in it.product(route.src, route.dst):
pipe = tuple([src] + route.pipe + [dst])
if pipe in pipes_chk: continue
for v in pipe:
if v not in pipes_valid:
log.fatal('Unknown route component (route: {}): {}'.format(route.name, v))
sys.exit(1)
pipes_chk.add(pipe) # to eliminate duplicates
pipes.setdefault(src, list()).append((dst, route.pipe))
log.noise('Pipelines (by src): {}'.format(pipes))
# Add reverse (obj -> name) mapping to relays
for name, relay_obj in relays.items(): relays[relay_obj] = name
self.relays, self.channels, self.routes = relays.copy(), channels.copy(), pipes
# Remove channels that aren't used in any of the routes
self.channel_map, channels = dict(), set()
for src, routes in self.routes.viewitems():
channels.add(src)
for dst, pipe in routes: channels.add(dst)
for channel in list(self.channels):
if channel not in channels:
log.debug('Ignoring channel, not used in any of the routes: {}'.format(channel))
del self.channels[channel]
else:
alias, channel = channel, self.channels[channel]
name = channel.get('name') or alias
self.channel_map[name] = alias