def test_lightweight_parse_includes(self):
# simple
cfg = NginxConfigParser(simple_config)
files = cfg.collect_all_files()
assert_that(files.keys(), equal_to([
'/amplify/test/fixtures/nginx/simple/conf.d/something.conf',
'/amplify/test/fixtures/nginx/simple/mime.types',
'/amplify/test/fixtures/nginx/simple/nginx.conf'
]))
# includes
cfg = NginxConfigParser(includes_config)
files = cfg.collect_all_files()
assert_that(files.keys(), equal_to([
'/amplify/test/fixtures/nginx/includes/conf.d/something.conf',
'/amplify/test/fixtures/nginx/includes/mime.types',
'/amplify/test/fixtures/nginx/includes/conf.d/additional.conf',
'/amplify/test/fixtures/nginx/includes/conf.d/include.conf',
'/amplify/test/fixtures/nginx/includes/nginx.conf'
]))
def test_lightweight_parse_includes_permissions(self):
"""
Checks that we get file permissions during lightweight parsing
"""
cfg = NginxConfigParser(simple_config)
files = cfg.collect_all_files()
test_file = '/amplify/test/fixtures/nginx/simple/conf.d/something.conf'
size = os.path.getsize(test_file)
mtime = int(os.path.getmtime(test_file))
permissions = oct(os.stat(test_file).st_mode & 0777)
assert_that(
files[test_file],
equal_to('%s_%s_%s' % (size, mtime, permissions))
)
class NginxConfig(object):
"""
Nginx config representation
Parses configs with all includes, etc
Main tasks:
- find all log formats
- find all access logs
- find all error logs
- find stub_status url
"""
def __init__(self, filename, binary=None, prefix=None):
self.filename = filename
self.binary = binary
self.prefix = prefix
self.log_formats = {}
self.access_logs = {}
self.error_logs = {}
self.test_errors = []
self.tree = {}
self.files = {}
self.index = []
self.ssl_certificates = {}
self.parser_errors = []
self.stub_status_urls = []
self.plus_status_external_urls = []
self.plus_status_internal_urls = []
self.parser = NginxConfigParser(filename)
def full_parse(self):
context.log.debug('parsing full tree of %s' % self.filename)
# parse raw data
self.parser.parse()
self.tree = self.parser.tree
self.files = self.parser.files
self.index = self.parser.index
self.parser_errors = self.parser.errors
# go through and collect all logical data
self.__recursive_search(subtree=self.parser.simplify())
# try to locate and use default logs (PREFIX/logs/*)
self.add_default_logs()
def get_all_files(self, include_ssl_certs=False):
"""
Goes through all files (light-parsed includes) and collects their mtime
:param include_ssl_certs: bool - include ssl certs or not
:return: {} - dict of files
"""
files = self.parser.collect_all_files(
include_ssl_certs=include_ssl_certs)
context.log.debug('found %s files for %s' %
(len(files.keys()), self.filename))
return files
def total_size(self):
"""
Returns the total size of a config tree
:return: int size in bytes
"""
result = 0
for file_name, file_info in self.files.iteritems():
result += file_info['size']
return result
def __recursive_search(self, subtree=None, ctx=None):
"""
Searches needed data in config's tree
:param subtree: dict with tree to parse
:param ctx: dict with context
"""
ctx = ctx if ctx is not None else {}
subtree = subtree if subtree is not None else {}
for key, value in subtree.iteritems():
if key == 'error_log':
error_logs = value if isinstance(value, list) else [value]
for er_log_definition in error_logs:
if er_log_definition == 'off':
continue
split_er_log_definition = er_log_definition.split(' ')
log_name = split_er_log_definition[0]
log_level = split_er_log_definition[-1] \
if split_er_log_definition[-1] in ERROR_LOG_LEVELS else 'error' # nginx default log level
log_name = re.sub('[\'"]', '',
log_name) # remove all ' and "
if log_name.startswith('syslog'):
continue
elif not log_name.startswith('/'):
log_name = '%s/%s' % (self.prefix, log_name)
if log_name not in self.error_logs:
self.error_logs[log_name] = log_level
elif key == 'access_log':
access_logs = value if isinstance(value, list) else [value]
for ac_log_definition in access_logs:
if ac_log_definition == 'off':
continue
parts = filter(lambda x: x, ac_log_definition.split(' '))
log_format = None if len(parts) == 1 else parts[1]
log_name = parts[0]
log_name = re.sub('[\'"]', '',
log_name) # remove all ' and "
if log_name.startswith('syslog'):
continue
elif not log_name.startswith('/'):
log_name = '%s/%s' % (self.prefix, log_name)
self.access_logs[log_name] = log_format
elif key == 'log_format':
for k, v in value.iteritems():
self.log_formats[k] = v
elif key == 'server' and isinstance(
value, list) and 'upstream' not in ctx:
for server in value:
current_ctx = copy.copy(ctx)
if server.get('listen') is None:
# if no listens specified, then use default *:80 and *:8000
listen = ['80', '8000']
else:
listen = server.get('listen')
listen = listen if isinstance(listen, list) else [listen]
ctx['ip_port'] = []
for item in listen:
listen_first_part = item.split(' ')[0]
addr, port = self.__parse_listen(listen_first_part)
if addr in ('*', '0.0.0.0'):
addr = '127.0.0.1'
elif addr == '[::]':
addr = '[::1]'
ctx['ip_port'].append((addr, port))
if 'server_name' in server:
ctx['server_name'] = server.get('server_name')
self.__recursive_search(subtree=server, ctx=ctx)
ctx = current_ctx
elif key == 'upstream':
for upstream, upstream_info in value.iteritems():
current_ctx = copy.copy(ctx)
ctx['upstream'] = upstream
self.__recursive_search(subtree=upstream_info, ctx=ctx)
ctx = current_ctx
elif key == 'location':
for location, location_info in value.iteritems():
current_ctx = copy.copy(ctx)
ctx['location'] = location
self.__recursive_search(subtree=location_info, ctx=ctx)
ctx = current_ctx
elif key == 'stub_status' and ctx and 'ip_port' in ctx:
for url in self.__status_url(ctx):
if url not in self.stub_status_urls:
self.stub_status_urls.append(url)
elif key == 'status' and ctx and 'ip_port' in ctx:
# use different url builders for external and internal urls
for url in self.__status_url(ctx, server_preferred=True):
if url not in self.plus_status_external_urls:
self.plus_status_external_urls.append(url)
# for internal (agent) usage local ip address is a better choice,
# because the external url might not be accessible from a host
for url in self.__status_url(ctx, server_preferred=False):
if url not in self.plus_status_internal_urls:
self.plus_status_internal_urls.append(url)
elif isinstance(value, dict):
self.__recursive_search(subtree=value, ctx=ctx)
elif isinstance(value, list):
for next_subtree in value:
if isinstance(next_subtree, dict):
self.__recursive_search(subtree=next_subtree, ctx=ctx)
@staticmethod
def __status_url(ctx, server_preferred=False):
"""
Creates stub/plus status url based on context
:param ctx: {} of current parsing context
:param server_preferred: bool - use server_name instead of listen
:return: [] of urls
"""
results = []
location = ctx.get('location', '/')
# remove all modifiers
location_parts = location.split(' ')
final_location_part = location_parts[-1]
# generate a random sting that will fit regex location
if location.startswith('~'):
try:
exact_location = rstr.xeger(final_location_part)
# check that regex location has / and add it
if not exact_location.startswith('/'):
exact_location = '/%s' % exact_location
except:
context.log.debug('bad regex location: %s' %
final_location_part)
exact_location = None
else:
exact_location = final_location_part
# if an exact location doesn't have / that's not a working location, we should not use it
if not exact_location.startswith('/'):
context.log.debug('bad exact location: %s' %
final_location_part)
exact_location = None
if exact_location:
for ip_port in ctx.get('ip_port'):
address, port = ip_port
if server_preferred and 'server_name' in ctx:
if isinstance(ctx['server_name'], list):
address = ctx['server_name'][0].split(' ')[0]
else:
address = ctx['server_name'].split(' ')[0]
results.append('%s:%s%s' % (address, port, exact_location))
return results
def run_test(self):
"""
Tests the configuration using nginx -t
Saves event info if syntax check was not successful
"""
start_time = time.time()
context.log.info('running %s -t -c %s' % (self.binary, self.filename))
if self.binary:
try:
_, nginx_t_err = subp.call("%s -t -c %s" %
(self.binary, self.filename),
check=False)
for line in nginx_t_err:
if 'syntax is' in line and 'syntax is ok' not in line:
self.test_errors.append(line)
except Exception as e:
exception_name = e.__class__.__name__
context.log.error('failed to %s -t -c %s due to %s' %
(self.binary, self.filename, exception_name))
context.log.debug('additional info:', exc_info=True)
end_time = time.time()
return end_time - start_time
def checksum(self):
"""
Calculates total checksum of all config files, certificates and permissions
:return: str checksum
"""
checksums = []
for filename in self.files.iterkeys():
checksums.append(hashlib.sha256(open(filename).read()).hexdigest())
for cert in self.ssl_certificates.iterkeys():
checksums.append(hashlib.sha256(open(cert).read()).hexdigest())
return hashlib.sha256('.'.join(checksums)).hexdigest()
def __parse_listen(self, listen):
"""
Parses listen directive value and return ip:port string, like *:80 and so on
:param listen: str raw listen
:return: str ip:port
"""
if '[' in listen:
# ipv6
addr_port_parts = filter(lambda x: x, listen.rsplit(']', 1))
address = '%s]' % addr_port_parts[0]
if len(addr_port_parts
) == 1: # only address specified, add default 80
return address, '80'
else: # get port
bracket, port = addr_port_parts[1].split(':')
return address, port
else:
# ipv4
addr_port_parts = filter(lambda x: x, listen.rsplit(':', 1))
if len(addr_port_parts) == 1:
# can be address or port only
is_port = addr_port_parts[0].isdigit()
if is_port: # port!
port = addr_port_parts[0]
return '*', port
else: # it was address only, add default 80
address = addr_port_parts[0]
return address, '80'
else:
address, port = addr_port_parts
return address, port
def add_default_logs(self):
"""
By default nginx uses logs placed in --prefix/logs/ directory
This method tries to find and add them
"""
access_log_path = '%s/logs/access.log' % self.prefix
if os.path.isfile(
access_log_path) and access_log_path not in self.access_logs:
self.access_logs[access_log_path] = None
error_log_path = '%s/logs/error.log' % self.prefix
if os.path.isfile(
error_log_path) and error_log_path not in self.error_logs:
self.error_logs[error_log_path] = 'error'
def run_ssl_analysis(self):
"""
Iterate over a list of ssl_certificate definitions and run ssl_analysis to construct a dictionary with
ssl_certificate value paired with results fo ssl_analysis.
:return: float run time
"""
if not self.parser.ssl_certificates:
return
start_time = time.time()
for cert_filename in set(self.parser.ssl_certificates):
if cert_filename not in self.ssl_certificates:
ssl_analysis_result = ssl_analysis(cert_filename)
if ssl_analysis_result:
self.ssl_certificates[cert_filename] = ssl_analysis_result
end_time = time.time()
return end_time - start_time
