def check_version_status(lint=False):
"""Check our version string in DNS for known issues and warn about them
the lookup should be <7 chars of commitid>.<patch>.<minor>.<major>.versioncheck.fuglu.org
in case of a release version, use 'release' instead of commit id
eg, the lookup for 0.6.3 would be:
release.3.6.0.versioncheck.fuglu.org
DNS will return NXDOMAIN or 127.0.0.<bitmask>
2: generic non security related issue
4: low risk security issue
8: high risk security issue
"""
bitmaskmap = {
2:
"there is a known (not security related) issue with this version - consider upgrading",
4:
"there is a known low-risk security issue with this version - an upgrade is recommended",
8:
"there is a known high-risk security issue with this version - upgrade as soon as possible!",
}
m = re.match(
r'^(?P<major>\d{1,4})\.(?P<minor>\d{1,4})\.(?P<patch>\d{1,4})(?:\-(?P<commitno>\d{1,4})\-g(?P<commitid>[a-f0-9]{7}))?$',
FUGLU_VERSION)
if m is None:
logging.warn("could not parse my version string %s" % FUGLU_VERSION)
return
parts = m.groupdict()
if 'commitid' not in parts or parts['commitid'] is None:
parts['commitid'] = 'release'
lookup = "{commitid}.{patch}.{minor}.{major}.versioncheck.fuglu.org".format(
**parts)
result = None
try:
result = socket.gethostbyname(lookup)
except Exception:
# DNS fails happen - try again next time
pass
if result is None:
return
ret = re.match(r'^127\.0\.0\.(?P<replycode>\d{1,4})$', result)
if ret is not None:
code = int(ret.groupdict()['replycode'])
for bitmask, message in bitmaskmap.items():
if code & bitmask == bitmask:
logging.warn(message)
if lint:
fc = FunkyConsole()
print(fc.strcolor(message, "yellow"))
def check_version_status(lint=False):
"""Check our version string in DNS for known issues and warn about them
the lookup should be <7 chars of commitid>.<patch>.<minor>.<major>.versioncheck.fuglu.org
in case of a release version, use 'release' instead of commit id
eg, the lookup for 0.6.3 would be:
release.3.6.0.versioncheck.fuglu.org
DNS will return NXDOMAIN or 127.0.0.<bitmask>
2: generic non security related issue
4: low risk security issue
8: high risk security issue
"""
bitmaskmap = {
2: "there is a known (not security related) issue with this version - consider upgrading",
4: "there is a known low-risk security issue with this version - an upgrade is recommended",
8: "there is a known high-risk security issue with this version - upgrade as soon as possible!",
}
m = re.match(
r"^(?P<major>\d{1,4})\.(?P<minor>\d{1,4})\.(?P<patch>\d{1,4})(?:\-(?P<commitno>\d{1,4})\-g(?P<commitid>[a-f0-9]{7}))?$",
FUGLU_VERSION,
)
if m == None:
logging.warn("could not parse my version string %s" % FUGLU_VERSION)
return
parts = m.groupdict()
if "commitid" not in parts or parts["commitid"] == None:
parts["commitid"] = "release"
lookup = "{commitid}.{patch}.{minor}.{major}.versioncheck.fuglu.org".format(**parts)
result = None
try:
result = socket.gethostbyname(lookup)
except:
# DNS fails happen - try again next time
pass
if result == None:
return
ret = re.match(r"^127\.0\.0\.(?P<replycode>\d{1,4})$", result)
if ret != None:
code = int(ret.groupdict()["replycode"])
for bitmask, message in bitmaskmap.items():
if code & bitmask == bitmask:
logging.warn(message)
if lint:
fc = FunkyConsole()
print(fc.strcolor(message, "yellow"))
def checkConfig(self):
"""Check if all required options without default are in the config file
Fill missing values with defaults if possible
"""
all_ok = True
fc = FunkyConsole()
for config, infodic in self.requiredvars.items():
section = infodic['section']
try:
var = self.config.get(section, config)
if 'validator' in infodic and not infodic["validator"](var):
print("Validation failed for [%s] :: %s" % (section, config))
all_ok = False
except configparser.NoSectionError:
print(fc.strcolor(f"Missing configuration section containing variables without default "
f"value [{section}] :: {config}", "red"))
all_ok = False
except configparser.NoOptionError:
print(fc.strcolor(f"Missing configuration value without default [{section}] :: {config}", "red"))
all_ok = False
# missing sections -> this is only a warning since section is not required
# as long as there are no required variables without default values...
if all_ok:
missingsections = set()
for config, infodic in self.requiredvars.items():
section = infodic['section']
if section not in missingsections and not self.config.has_section(section):
missingsections.add(section)
for section in missingsections:
print(fc.strcolor(f"Missing configuration section [{section}] :: "
f"All variables will use default values", "yellow"))
return all_ok
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "":
if not os.path.isdir(trashdir):
print(
fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
# sql config override
sqlconfigdbconnectstring = self.config.get(
'databaseconfig', 'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print("")
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null')
sqlvars = dict(
section='testsection', option='testoption', scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
allplugins = self.plugins + self.prependers + self.appenders
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
errors = errors + 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % errors)
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
newsections = newconfig.sections()
# values that usually differ from the default
excludelist = [
('main', 'plugins'),
('main', 'prependers'),
('main', 'identifier'),
('main', 'appenders'),
('main', 'trashdir'),
('ArchivePlugin', 'archivedir'),
]
for newsection in newsections:
# print "Checking section %s"%newsection
if not currentconfig.has_section(newsection):
print "%s: section '%s' is missing in your current config" % (fc.strcolor('MISSING SECTION', 'red'), fc.strcolor(newsection, 'cyan'))
continue
newitems = newconfig.options(newsection)
currentitems = currentconfig.options(newsection)
toomanyitems = set(currentitems) - set(newitems)
if len(toomanyitems) > 0:
for item in toomanyitems:
print "%s: Your option '%s' in section '%s' is not known in new config" % (fc.strcolor('UNKNOWN OPTION', 'yellow'), fc.strcolor(item, 'cyan'), fc.strcolor(newsection, 'cyan'))
for key in newitems:
defaultvalue = newconfig.get(newsection, key)
if not currentconfig.has_option(newsection, key):
print "%s: add option '%s' in section '%s'. Default value is: '%s'" % (fc.strcolor('MISSING OPTION', 'red'), fc.strcolor(key, 'cyan'), fc.strcolor(newsection, 'cyan'), fc.strcolor(defaultvalue, 'cyan'))
continue
# values that usually differ from the default
excludelist = [
('main', 'plugins'),
('main', 'prependers'),
('main', 'identifier'),
('main', 'appenders'),
('main', 'trashdir'),
('ArchivePlugin', 'archivedir'),
]
for newsection in newsections:
# print "Checking section %s"%newsection
if not currentconfig.has_section(newsection):
print("%s: section '%s' is missing in your current config" %
(fc.strcolor('MISSING SECTION',
'red'), fc.strcolor(newsection, 'cyan')))
continue
newitems = newconfig.options(newsection)
currentitems = currentconfig.options(newsection)
toomanyitems = set(currentitems) - set(newitems)
if len(toomanyitems) > 0:
for item in toomanyitems:
print(
"%s: Your option '%s' in section '%s' is not known in new config"
% (fc.strcolor('UNKNOWN OPTION', 'yellow'),
fc.strcolor(item, 'cyan'), fc.strcolor(newsection, 'cyan')))
for key in newitems:
defaultvalue = newconfig.get(newsection, key)
# values that usually differ from the default
excludelist = [
('main', 'plugins'),
('main', 'prependers'),
('main', 'identifier'),
('main', 'appenders'),
('main', 'trashdir'),
('ArchivePlugin', 'archivedir'),
]
for newsection in newsections:
# print "Checking section %s"%newsection
if not currentconfig.has_section(newsection):
print("%s: section '%s' is missing in your current config" % (
fc.strcolor('MISSING SECTION', 'red'), fc.strcolor(newsection, 'cyan')))
continue
newitems = newconfig.options(newsection)
currentitems = currentconfig.options(newsection)
toomanyitems = set(currentitems) - set(newitems)
if len(toomanyitems) > 0:
for item in toomanyitems:
print("%s: Your option '%s' in section '%s' is not known in new config" % (fc.strcolor(
'UNKNOWN OPTION', 'yellow'), fc.strcolor(item, 'cyan'), fc.strcolor(newsection, 'cyan')))
for key in newitems:
defaultvalue = newconfig.get(newsection, key)
if not currentconfig.has_option(newsection, key):
print("%s: add option '%s' in section '%s'. Default value is: '%s'" % (fc.strcolor('MISSING OPTION',
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "" and not os.path.isdir(trashdir):
print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
# sql config override
sqlconfigdbconnectstring = self.config.get('databaseconfig',
'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print()
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
sqlvars = dict(section='testsection',
option='testoption',
scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
allplugins = self.plugins + self.prependers + self.appenders
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
errors = errors + 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % errors)
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
errors +=1
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
errors += 1
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "" and not os.path.isdir(trashdir):
print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
errors += 1
# sql config override
sqlconfigdbconnectstring = self.config.get('databaseconfig', 'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print()
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null',
att_cachelimit=self.config.getint('performance','att_mgr_cachesize'))
sqlvars = dict(
section='testsection', option='testoption', scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
errors += 1
allplugins = self.plugins + self.prependers + self.appenders
perrors = 0
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
perrors += 1
errors += 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % perrors)
if "milter" in self.config.get('main', 'incomingport') \
and self.config.get('performance', 'backend') != 'process':
try:
minfreethreads = self.config.getint('performance', 'minfreethreads')
if minfreethreads < 1:
print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads < 1"', 'yellow'))
print("To keep milter responsive it is recommended to set minfreethreads >= 1\n"
"to make fuglu more resonsive.\n")
except (configparser.NoSectionError, configparser.NoOptionError):
print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads is not defined!"', 'yellow'))
print("To keep fuglu-milter responsive it is recommended to set minfreethreads >= 1\n")
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
return errors
def check_version_status(lint=False):
import pkg_resources
"""Check our version string in DNS for known issues and warn about them
the lookup should be <7 chars of commitid>.<patch>.<minor>.<major>.versioncheck.fuglu.org
in case of a release version, use 'release' instead of commit id
lookup examples:
- 0.9.1 -> 0.release.1.9.0.versioncheck.fuglu.org
- 0.9.1rc5 -> 5.rc.1.9.0.versioncheck.fuglu.org
- 0.9.1.dev20181204183236 -> 20181204183236.dev.1.9.0.versioncheck.fuglu.org
DNS will return NXDOMAIN or 127.0.0.<bitmask>
2: generic non security related issue
4: low risk security issue
8: high risk security issue
"""
bitmaskmap = {
2: "there is a known (not security related) issue with this version - consider upgrading",
4: "there is a known low-risk security issue with this version - an upgrade is recommended",
8: "there is a known high-risk security issue with this version - upgrade as soon as possible!",
}
try:
version_obj = pkg_resources.parse_version(FUGLU_VERSION)._version
if not isinstance(version_obj, pkg_resources.extern.packaging.version._Version):
version_obj = None
except AttributeError:
# Older versions of setuptools (< 20.2.2) return a tuple.
# Accessing "_version" will raise an attribute error.
version_obj = None
except Exception as e:
# Skip test for other problems...
logging.getLogger("fuglu.check_version_status").warning(str(e))
version_obj = None
if version_obj is None:
logging.getLogger("fuglu.check_version_status")\
.warning("Version string %s could not be parsed to pkg_resources version object", FUGLU_VERSION)
return
(major, minor, micro) = version_obj.release
(stage_str, stage_id) = ('release', 0)
if version_obj.dev:
stage_str = version_obj.dev[0]
stage_id = version_obj.dev[1]
if version_obj.pre:
stage_str = version_obj.pre[0]
stage_id = version_obj.pre[1]
if version_obj.post:
stage_str = version_obj.post[0]
stage_id = version_obj.post[1]
parts = {
'major': major,
'minor': minor,
'micro': micro,
'stage_str': stage_str,
'stage_id': stage_id
}
lookup = "{stage_id}.{stage_str}.{micro}.{minor}.{major}.versioncheck.fuglu.org".format(**parts)
result = None
try:
result = socket.gethostbyname(lookup)
except Exception:
# DNS fails happen - try again next time
pass
if result is None:
return
ret = re.match(r'^127\.0\.0\.(?P<replycode>\d{1,4})$', result)
if ret is not None:
code = int(ret.groupdict()['replycode'])
for bitmask, message in bitmaskmap.items():
if code & bitmask == bitmask:
logging.warn(message)
if lint:
fc = FunkyConsole()
print(fc.strcolor(message, "yellow"))