def _traverse(self, key, value, path="/", named_chain=[],
match_action=None):
generic_path = self._generalize_path(path)
if self._is_leaf(generic_path):
if self._path_action.get(generic_path, None) != match_action:
return
expanded_pattern = self._expand_pattern(value, generic_path)
yield generic_path, named_chain, expanded_pattern
return
for k, v in self._as_assoc(value):
if not self._is_assoc(v):
continue
next_path = normalize_path(path + "/" + str(k))
next_generic_path = self._generalize_path(next_path)
if "name" in v and not self._is_leaf(next_generic_path):
next_name = [v]
else:
next_name = []
for expansion_node in self._traverse(k, v, path=next_path,
named_chain=named_chain + \
next_name,
match_action=match_action):
yield expansion_node
def _traverse(self,
key,
value,
path="/",
named_chain=[],
match_action=None):
generic_path = self._generalize_path(path)
if self._is_leaf(generic_path):
if self._path_action.get(generic_path, None) != match_action:
return
expanded_pattern = self._expand_pattern(value, generic_path)
yield generic_path, named_chain, expanded_pattern
return
for k, v in self._as_assoc(value):
if not self._is_assoc(v):
continue
next_path = normalize_path(path + "/" + str(k))
next_generic_path = self._generalize_path(next_path)
if "name" in v and not self._is_leaf(next_generic_path):
next_name = [v]
else:
next_name = []
for expansion_node in self._traverse(k, v, path=next_path,
named_chain=named_chain + \
next_name,
match_action=match_action):
yield expansion_node
def hooks_create(srcdir, lnkdir, names):
"""
Offer to create hook links if they are missing
"""
srcndir = util.normalize_path(srcdir)
lnkndir = util.normalize_path(lnkdir)
src_l = [_ for _ in glob.glob(os.path.join(srcndir, "*"))
if os.path.isfile(_)]
if names == []:
for hook in src_l:
link = hook_exists(hook, lnkndir)
if link:
print("{0} already has link {1}"
''.format(os.path.basename(hook),
link))
else:
print("{0} does not appear to have a link "
"in the link directory"
"".format(os.path.basename(hook)))
answer = raw_input("Shall I add one? > ")
if answer.strip().lower().startswith('y'):
os.symlink(hook,
os.path.join(lnkndir, os.path.basename(hook)))
else:
for name in names:
path = os.path.join(srcndir, name)
if path not in src_l:
print("I don't see a hook script named {0} to link"
''.format(name))
continue
link = hook_exists(path, lnkndir)
if link:
print("{0} already has link {1}"
''.format(os.path.basename(path), link))
else:
os.symlink(path, os.path.join(lnkndir, os.path.basename(path)))
def hook_exists(path, linkdir):
"""
Return True if a link in linkdir points at path, else False.
We assume linkdir is already normalized.
"""
rval = ''
for link in [_ for _ in glob.glob(os.path.join(linkdir, "*"))
if os.path.islink(_)]:
targ = util.normalize_path(os.readlink(link))
if targ == path:
rval = link
break
return rval
def __init__(self,
callback,
conf_root,
listen_ipv4=None,
listen_ipv6=None,
default_port=None,
listen_ipv4_ssl=None,
listen_ipv6_ssl=None,
default_port_ssl=None,
pemfile=None,
cacert=None):
'''Initializes the server and listens on the specified addresses.
param callback : callable
The callable that is called as callabale(msg, addr) whenever a message is received.
param conf_root : unicode
Path to the root of the configuration tree
param listen_ipv4 : comma separated basestring of addresses
Addresses to listen on for TCP and UDP connections
param listen_ipv6 : comma separated basestring of addresses
Addresses to listen on for TCP and UDP connections
param default_port : short
Port to use for listen_ipv4 and listen_ipv6 if they don't specify a custom port
param listen_ipv4_ssl : comma separated basestring of addresses
Addresses to listen on for SSL/TLS connections
param listen_ipv6_ssl : comma separated basestring of addresses
Addresses to listen on for SSL/TLS connections
param default_port_ssl : short
Port to use for listen_ipv4_ssl and listen_ipv6_ssl if they don't specify a custom port
param pemfile : unicode
File path to the pem file containing private and public keys for SSL/TLS
param cacert : unicode
File path to the cacert file with which the public key in pemfile is signed
'''
self.log = logging.getLogger('server') # internal logger
self.callback = callback
self.closed = False
self.stream_socks = set()
self.ssl_socks = set()
self.dgram_socks = set()
self.client_stream_socks = set()
self.stream_buffers = {}
self.client_socket_addrs = {}
self.select_timeout = None # Set on shutdown to prevent infinite wait
self.pemfile = normalize_path(
pemfile if pemfile is not None else options.server.pemfile,
conf_root)
self.cacert = normalize_path(
cacert if cacert is not None else options.server.cacert, conf_root)
# Initialize server sockets
listen_ipv4 = listen_ipv4 if listen_ipv4 is not None else options.server.listen_ipv4
listen_ipv6 = listen_ipv6 if listen_ipv6 is not None else options.server.listen_ipv6
default_port = default_port if default_port is not None else options.server.default_port
ipv4_addrs = parse_addrs(listen_ipv4, default_port)
ipv6_addrs = parse_addrs(listen_ipv6, default_port)
for addr in ipv4_addrs:
self.stream_socks.add(
self.connect(addr, socket.AF_INET, socket.SOCK_STREAM))
self.dgram_socks.add(
self.connect(addr, socket.AF_INET, socket.SOCK_DGRAM))
for addr in ipv6_addrs:
self.stream_socks.add(
self.connect(addr, socket.AF_INET6, socket.SOCK_STREAM))
self.dgram_socks.add(
self.connect(addr, socket.AF_INET6, socket.SOCK_DGRAM))
# Same thing for SSL addresses, except not datagram sockets
listen_ipv4_ssl = listen_ipv4_ssl if listen_ipv4_ssl is not None else options.server.listen_ipv4_ssl
listen_ipv6_ssl = listen_ipv6_ssl if listen_ipv6_ssl is not None else options.server.listen_ipv6_ssl
default_port_ssl = default_port_ssl if default_port_ssl is not None else options.server.default_port_ssl
self.validate_ssl_config(listen_ipv4_ssl, listen_ipv6_ssl)
ipv4_addrs_ssl = parse_addrs(listen_ipv4_ssl, default_port_ssl)
ipv6_addrs_ssl = parse_addrs(listen_ipv6_ssl, default_port_ssl)
for addr in ipv4_addrs_ssl:
s = self.connect(addr, socket.AF_INET, socket.SOCK_STREAM, True)
self.stream_socks.add(s)
self.ssl_socks.add(s)
for addr in ipv6_addrs_ssl:
s = self.connect(addr, socket.AF_INET6, socket.SOCK_STREAM, True)
self.stream_socks.add(s)
self.ssl_socks.add(s)
self.all_socks = set(self.stream_socks | self.dgram_socks)
def main():
try:
init_options()
except OptionsError as e:
sys.stderr.write("Error: {0}\n\n".format(e))
sys.stderr.write(usage())
sys.stderr.write("\n");
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.generate_config:
print(generate_sample_config()) # XXX: this doesn't yet work properly because of groper
sys.exit()
conf_root = os.path.dirname(os.path.abspath(options.main.config))
facility_db = FacilityDB()
try:
facility_db.load_config(normalize_path(options.main.facilities_config, conf_root))
except (IOError) as e:
sys.stderr.write("Error reading {0}: {1}\n".format(options.main.facilities_config, e))
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
except (FacilityError, configparser.Error) as e:
sys.stderr.write("{0} contains errors:\n\n".format(options.main.facilities_config))
if hasattr(e, 'lineno'):
e = 'Error on line {0}: {1}'.format(e.lineno, e.message.split("\n")[0].strip())
sys.stderr.write("{0}\n\n".format(e))
sys.stderr.write("Exiting now.\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.check_config:
sys.exit() # We are just checking the config file, so exit here.
cache_config_checksum()
create_dirs()
if options.main.daemon:
daemonize()
if options.main.user:
drop_privileges(options.main.user)
if options.main.pidfile:
write_pid(options.main.pidfile)
atexit.register(exit_handler)
setup_logging()
try:
logging.getLogger().info("Starting loghogd.")
compressor = Compressor()
compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+')
writer = Writer(facility_db, compressor, options.main.logdir)
processor = Processor(facility_db, writer)
server = Server(processor.on_message, conf_root)
signal_handler = make_shutdown_handler(server, writer, compressor)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer))
except Exception as e:
logging.getLogger().error(e)
logging.getLogger().error('Exiting abnormally due to an error at startup.')
sys.exit(os.EX_CONFIG)
try:
compressor.start()
server.run()
except Exception as e:
logging.getLogger().exception(e)
logging.getLogger().error('Exiting abnormally due to an error at runtime.')
shutdown(None, server, writer, compressor)
sys.exit(os.EX_SOFTWARE)
logging.getLogger().info('Shutdown complete. Exiting.')
def main():
try:
init_options()
except OptionsError as e:
sys.stderr.write("Error: {0}\n\n".format(e))
sys.stderr.write(usage())
sys.stderr.write("\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.generate_config:
print(generate_sample_config()
) # XXX: this doesn't yet work properly because of groper
sys.exit()
conf_root = os.path.dirname(os.path.abspath(options.main.config))
facility_db = FacilityDB()
try:
facility_db.load_config(
normalize_path(options.main.facilities_config, conf_root))
except (IOError) as e:
sys.stderr.write("Error reading {0}: {1}\n".format(
options.main.facilities_config, e))
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
except (FacilityError, configparser.Error) as e:
sys.stderr.write("{0} contains errors:\n\n".format(
options.main.facilities_config))
if hasattr(e, 'lineno'):
e = 'Error on line {0}: {1}'.format(
e.lineno,
e.message.split("\n")[0].strip())
sys.stderr.write("{0}\n\n".format(e))
sys.stderr.write("Exiting now.\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.check_config:
sys.exit() # We are just checking the config file, so exit here.
cache_config_checksum()
create_dirs()
if options.main.daemon:
daemonize()
if options.main.user:
drop_privileges(options.main.user)
if options.main.pidfile:
write_pid(options.main.pidfile)
atexit.register(exit_handler)
setup_logging()
try:
logging.getLogger().info("Starting loghogd.")
compressor = Compressor()
compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+')
writer = Writer(facility_db, compressor, options.main.logdir)
processor = Processor(facility_db, writer)
server = Server(processor.on_message, conf_root)
signal_handler = make_shutdown_handler(server, writer, compressor)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer))
except Exception as e:
logging.getLogger().error(e)
logging.getLogger().error(
'Exiting abnormally due to an error at startup.')
sys.exit(os.EX_CONFIG)
try:
compressor.start()
server.run()
except Exception as e:
logging.getLogger().exception(e)
logging.getLogger().error(
'Exiting abnormally due to an error at runtime.')
shutdown(None, server, writer, compressor)
sys.exit(os.EX_SOFTWARE)
logging.getLogger().info('Shutdown complete. Exiting.')
def __init__(self, callback, conf_root, listen_ipv4=None, listen_ipv6=None, default_port=None, listen_ipv4_ssl=None, listen_ipv6_ssl=None, default_port_ssl=None, pemfile=None, cacert=None):
'''Initializes the server and listens on the specified addresses.
param callback : callable
The callable that is called as callabale(msg, addr) whenever a message is received.
param conf_root : unicode
Path to the root of the configuration tree
param listen_ipv4 : comma separated basestring of addresses
Addresses to listen on for TCP and UDP connections
param listen_ipv6 : comma separated basestring of addresses
Addresses to listen on for TCP and UDP connections
param default_port : short
Port to use for listen_ipv4 and listen_ipv6 if they don't specify a custom port
param listen_ipv4_ssl : comma separated basestring of addresses
Addresses to listen on for SSL/TLS connections
param listen_ipv6_ssl : comma separated basestring of addresses
Addresses to listen on for SSL/TLS connections
param default_port_ssl : short
Port to use for listen_ipv4_ssl and listen_ipv6_ssl if they don't specify a custom port
param pemfile : unicode
File path to the pem file containing private and public keys for SSL/TLS
param cacert : unicode
File path to the cacert file with which the public key in pemfile is signed
'''
self.log = logging.getLogger('server') # internal logger
self.callback = callback
self.closed = False
self.stream_socks = set()
self.ssl_socks = set()
self.dgram_socks = set()
self.client_stream_socks = set()
self.stream_buffers = {}
self.client_socket_addrs = {}
self.select_timeout = None # Set on shutdown to prevent infinite wait
self.pemfile = normalize_path(pemfile if pemfile is not None else options.server.pemfile, conf_root)
self.cacert = normalize_path(cacert if cacert is not None else options.server.cacert, conf_root)
# Initialize server sockets
listen_ipv4 = listen_ipv4 if listen_ipv4 is not None else options.server.listen_ipv4
listen_ipv6 = listen_ipv6 if listen_ipv6 is not None else options.server.listen_ipv6
default_port = default_port if default_port is not None else options.server.default_port
ipv4_addrs = parse_addrs(listen_ipv4, default_port)
ipv6_addrs = parse_addrs(listen_ipv6, default_port)
for addr in ipv4_addrs:
self.stream_socks.add(self.connect(addr, socket.AF_INET, socket.SOCK_STREAM))
self.dgram_socks.add(self.connect(addr, socket.AF_INET, socket.SOCK_DGRAM))
for addr in ipv6_addrs:
self.stream_socks.add(self.connect(addr, socket.AF_INET6, socket.SOCK_STREAM))
self.dgram_socks.add(self.connect(addr, socket.AF_INET6, socket.SOCK_DGRAM))
# Same thing for SSL addresses, except not datagram sockets
listen_ipv4_ssl = listen_ipv4_ssl if listen_ipv4_ssl is not None else options.server.listen_ipv4_ssl
listen_ipv6_ssl = listen_ipv6_ssl if listen_ipv6_ssl is not None else options.server.listen_ipv6_ssl
default_port_ssl = default_port_ssl if default_port_ssl is not None else options.server.default_port_ssl
self.validate_ssl_config(listen_ipv4_ssl, listen_ipv6_ssl)
ipv4_addrs_ssl = parse_addrs(listen_ipv4_ssl, default_port_ssl)
ipv6_addrs_ssl = parse_addrs(listen_ipv6_ssl, default_port_ssl)
for addr in ipv4_addrs_ssl:
s = self.connect(addr, socket.AF_INET, socket.SOCK_STREAM, True)
self.stream_socks.add(s)
self.ssl_socks.add(s)
for addr in ipv6_addrs_ssl:
s = self.connect(addr, socket.AF_INET6, socket.SOCK_STREAM, True)
self.stream_socks.add(s)
self.ssl_socks.add(s)
self.all_socks = set(self.stream_socks | self.dgram_socks)