def __init_db__(self):
plugin_data = {
'id': self.__id if self.__id else '',
'type': self.__type if self.__type else '',
'description': self.__description if self.__description else ''
}
self.__host = Wildcard.av_config(
self.__config_file.get(
'properties', 'host')) if self.__config_file.get(
'properties',
'host') else self.__alienvault_config['dbhost']
self.__user = Wildcard.av_config(
self.__config_file.get(
'properties', 'user')) if self.__config_file.get(
'properties',
'user') else self.__alienvault_config['dbuser']
self.__password = Wildcard.av_config(
self.__config_file.get(
'properties', 'password')) if self.__config_file.get(
'properties',
'password') else self.__alienvault_config['dbpass']
self.__database = self.__config_file.get('properties', 'database')
try:
self.__db_conn = MySQLdb.connect(host=self.__host,
user=self.__user,
passwd=self.__password,
db=self.__database)
self.__db_cursor = self.__db_conn.cursor()
except Exception as e:
raise PluginError(msg='Cannot connect to database: %s' % e,
plugin=self.__name,
plugin_data=plugin_data)
def __check_set_conditions__(self, values):
(outcome, description, fo) = (True, '', '')
values = filter(None, values)
values_set = set(values)
values_set_str = str(values_set)
for condition in self.__conditions['set']:
(operation, parsed_condition) = Wildcard.set_operation(condition)
if operation is None or parsed_condition is None:
raise CheckError(msg='Unknown set operation: "%s"' %
str(condition),
plugin=self.__plugin.get_name())
try:
diff = eval(values_set_str + parsed_condition)
if diff != set():
set_list = ", ".join(
str(elem) for elem in diff if elem != '')
if self.__plugin.get_alienvault_config(
)['has_ha'] and self.__ha_dependant:
description += '\n\tCondition "%s: %s" failed because HA is enabled. Ignoring it' % (
str(operation), set_list)
fo = self.__formatted_output.replace(
'@set_list@', set_list)
else:
description += '\n\tOffending values for operation "%s": %s\n' % (
str(operation), set_list)
fo = self.__formatted_output.replace(
'@set_list@', set_list)
outcome = False
except SyntaxError, msg:
raise CheckError(msg='Invalid syntax',
plugin=self.__plugin.get_name())
def __check_set_conditions__(self, values):
(outcome, description, fo) = (True, '', '')
values = filter(None, values)
values_set = set(values)
values_set_str = str(values_set)
for condition in self.__conditions['set']:
(operation, parsed_condition) = Wildcard.set_operation(condition)
if operation is None or parsed_condition is None:
raise CheckError(msg='Unknown set operation: "%s"' % str(condition),
plugin=self.__plugin.get_name())
try:
diff = eval(values_set_str + parsed_condition)
if diff != set():
set_list = ", ".join(str(elem) for elem in diff if elem != '')
if self.__plugin.get_alienvault_config()['has_ha'] and self.__ha_dependant:
description += '\n\tCondition "%s: %s" failed because HA is enabled. Ignoring it' % (str(operation), set_list)
fo = self.__formatted_output.replace('@set_list@', set_list)
else:
description += '\n\tOffending values for operation "%s": %s\n' % (str(operation), set_list)
fo = self.__formatted_output.replace('@set_list@', set_list)
outcome = False
except SyntaxError, msg:
raise CheckError(msg='Invalid syntax',
plugin=self.__plugin.get_name())
def __init_db__(self):
plugin_data = {'id': self.__id if self.__id else '',
'type': self.__type if self.__type else '',
'description': self.__description if self.__description else ''}
self.__host = Wildcard.av_config(self.__config_file.get('properties', 'host')) if self.__config_file.get('properties', 'host') else self.__alienvault_config['dbhost']
self.__user = Wildcard.av_config(self.__config_file.get('properties', 'user')) if self.__config_file.get('properties', 'user') else self.__alienvault_config['dbuser']
self.__password = Wildcard.av_config(self.__config_file.get('properties', 'password')) if self.__config_file.get('properties', 'password') else self.__alienvault_config['dbpass']
self.__database = self.__config_file.get('properties', 'database')
try:
self.__db_conn = MySQLdb.connect(host=self.__host, user=self.__user, passwd=self.__password, db=self.__database)
self.__db_cursor = self.__db_conn.cursor()
except Exception as e:
raise PluginError(msg='Cannot connect to database: %s' % e,
plugin=self.__name,
plugin_data=plugin_data)
def tidy(directory: str, collection: dict):
ignoring_file = '.rmdupignore'
wildcard = Wildcard()
for dirpath, _, filenames in os.walk(directory):
wildcard.append(dirpath, ignoring_file)
if wildcard.match(dirpath):
continue
path = PurePath(dirpath)
for filename in filenames:
if filename == ignoring_file or filename == 'Icon\r':
continue
filepath = str(path.joinpath(filename))
# ignore
if wildcard.match(filepath):
continue
f = FileInfo(filepath)
if isinstance(f, FileInfo):
logging.debug(f.uri)
collection.setdefault(f, []).append(str(f))
else:
logging.warning(f)
def __generate_blocked_output(self, config_file, plugin, plugin_data, sections, error_msg):
# Parse the plugin configuration file.
checks = {}
if config_file:
try:
data = {'filename': '', 'command': ''}
for section in sections:
if section != 'properties':
pass
else:
items = dict(config_file.items(section))
if 'type' in plugin_data.keys() and plugin_data['type'] == "file":
data['filename'] = items['filename']
elif plugin_data and plugin_data['type'] == "command":
data['command'] = items['command']
break
for section in sections:
if section != 'properties':
items = dict(config_file.items(section))
aux_app_type = []
for x in items['appliance_type'].split(','):
aux_app_type += Wildcard.appliance_exec(x.strip())
if self.__alienvault_config['hw_profile'].lower() not in aux_app_type:
continue
if 'type' in plugin_data.keys() and plugin_data['type'] == "db":
try:
data['command'] = "echo '%s;' | ossim-db" % items['query']
except Exception:
pass
elif 'type' in plugin_data.keys() and plugin_data['type'] == "file":
try:
data['command'] = "cat %s" % data['filename']
except Exception:
pass
checks[section] = {'result': 'blocked',
'detail': error_msg,
'description': items['description'],
'command': data['command'],
'severity': items['severity']}
# No checks for this plugin in the current appliance --> do not return plugin details
if not checks:
return
except Exception as e:
checks = {}
self.__summary[plugin] = {'result': 'blocked',
'summary': error_msg,
'checks': checks}
def __init_command__(self):
plugin_data = {'id': self.__id if self.__id else '',
'type': self.__type if self.__type else '',
'description': self.__description if self.__description else ''}
self.__command = self.__config_file.get('properties', 'command')
self.__command = Wildcard.av_config(self.__command)
try:
proc = subprocess.Popen(self.__command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.__data, err = proc.communicate()
self.__data_len = len(self.__data)
except Exception as e:
raise PluginError(msg='Cannot run command "%s": %s' % (self.__command, e),
plugin=self.__name,
plugin_data=plugin_data)
def __init_command__(self):
plugin_data = {'id': self.__id if self.__id else '',
'type': self.__type if self.__type else '',
'description': self.__description if self.__description else ''}
self.__command = self.__config_file.get('properties', 'command')
self.__command = Wildcard.av_config(self.__command)
try:
proc = subprocess.Popen(self.__command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.__data, err = proc.communicate()
self.__data_len = len(self.__data)
except Exception as e:
raise PluginError(msg='Cannot run command "%s": %s' % (self.__command, e),
plugin=self.__name,
plugin_data=plugin_data)
def __run_hw__(self):
(outcome, description, fo) = (True, '', '')
# Check hardware requirements.
for hw_req in self.__hw_list.split(','):
(hw_req_pretty, eval_str) = Wildcard.hw_config(hw_req)
if not eval(eval_str):
outcome = False
description += "\n\t %s" % hw_req_pretty
aux = re.findall(r'(\S+)(>=|==|<=)(\S+)', eval_str)
# fo += "%s: Condition (available vs expected) --> %s;" % (hw_req_pretty, eval_str)
fo += "%s: Expected a value %s %s, but %s found; " % (hw_req_pretty,
aux[0][1],
aux[0][2],
aux[0][0])
return (outcome, description, fo)
def test_wildcard(monkeypatch):
read_data = '''
.git/
*/.git/
*.jpg
*.py
a/*/c/
*/mame0184-64bit/
'''
with patch('io.open', mock_open(read_data=read_data)):
wildcard = Wildcard('.', '.rmdupignore')
assert wildcard.match('.git')
assert wildcard.match('.git/a/b/c')
assert wildcard.match('/a/b/c/.git/')
assert wildcard.match('/a/b/c/.git/d/e/f')
assert wildcard.match('/a/b/c/d.e.f.jpg')
def __run_hw__(self):
(outcome, description, fo) = (True, '', '')
# Check hardware requirements.
for hw_req in self.__hw_list.split(','):
(hw_req_pretty, eval_str) = Wildcard.hw_config(hw_req)
if not eval(eval_str):
outcome = False
description += "\n\t %s" % hw_req_pretty
if hw_req == '@is_vm@':
fo += "%s; " % hw_req_pretty
break
else:
aux = re.findall(r'(\S+)(>=|==|<=)(\S+)', eval_str)
# fo += "%s: Condition (available vs expected) --> %s;" % (hw_req_pretty, eval_str)
fo += "%s: Expected a value %s %s, but %s found; " % (hw_req_pretty,
aux[0][1],
aux[0][2],
aux[0][0])
return (outcome, description, fo)
def __generate_blocked_output(self, config_file, plugin, plugin_data,
sections, error_msg):
# Parse the plugin configuration file.
checks = {}
if config_file:
try:
data = {'filename': '', 'command': ''}
for section in sections:
if section != 'properties':
pass
else:
items = dict(config_file.items(section))
if 'type' in plugin_data.keys(
) and plugin_data['type'] == "file":
data['filename'] = items['filename']
elif plugin_data and plugin_data['type'] == "command":
data['command'] = items['command']
break
for section in sections:
if section != 'properties':
items = dict(config_file.items(section))
aux_app_type = []
for x in items['appliance_type'].split(','):
aux_app_type += Wildcard.appliance_exec(x.strip())
if self.__alienvault_config['hw_profile'].lower(
) not in aux_app_type:
continue
if 'type' in plugin_data.keys(
) and plugin_data['type'] == "db":
try:
data[
'command'] = "echo '%s;' | ossim-db" % items[
'query']
except Exception:
pass
elif 'type' in plugin_data.keys(
) and plugin_data['type'] == "file":
try:
data['command'] = "cat %s" % data['filename']
except Exception:
pass
checks[section] = {
'result': 'blocked',
'detail': error_msg,
'description': items['description'],
'command': data['command'],
'severity': items['severity']
}
# No checks for this plugin in the current appliance --> do not return plugin details
if not checks:
return
except Exception as e:
checks = {}
self.__summary[plugin] = {
'result': 'blocked',
'summary': error_msg,
'checks': checks
}
def __init__(self, plugin, section):
# 'check' properties.
self.__name = ''
self.__type = ''
self.__pattern = ''
self.__category = ''
self.__description = ''
self.__summary_passed = ''
self.__summary_failed = ''
self.__remediation = ''
self.__plugin = None
# 'file' type checks only.
self.__checksums = []
# 'file' and 'command' checks.
self.__regex = None
# 'db' type checks.
self.__query = ''
self.__pivot = False
self.__introduced = ''
self.__output = ''
self.__formatted_output = ''
self.__appliance_type = []
self.__fail_if_empty = True
self.__fail_if_empty_output = ''
self.__fail_only_if_all_failed = False
self.__split_by_comma = False
self.__ha_dependant = False
self.__severity = 'Warning'
self.__conditions = {'basic': [], 'set': []}
self.__actions = []
self.__aux_data = {}
self.__strike_zone = False
self.__version_type = ''
config_file = plugin.get_config_file()
self.__name = section
self.__plugin = plugin
# Parse section options.
# Different sections or check 'types' are mutually exclusive.
items = config_file.items(section)
try:
# Retrieve first the formatted_output field
for (name, value) in items:
if name == 'formatted_output':
self.__formatted_output = value.replace("{nl}", "\n")
items.remove((name, value))
break
# Now the rest
for (name, value) in items:
if name == 'checksum':
self.__type = name
self.__checksums = [
tuple(x.split(':')) for x in value.split(';')
]
elif name == 'pattern':
self.__type = name
self.__pattern = str(value)
value = Wildcard.av_config(value, escape=True)
self.__regex = re.compile(value, re.MULTILINE)
elif name == 'query':
self.__type = name
if value.startswith("@pivot@:"):
self.__query = value[8:]
self.__pivot = True
else:
self.__query = value
self.__query = Wildcard.av_config(self.__query,
escape=True)
elif name == 'hardware':
self.__type = name
self.__hw_list = value
elif name == 'category':
self.__category = value
elif name == 'fail_if_empty':
if value in ['True', 'False']:
self.__fail_if_empty = eval(value)
elif name == 'fail_if_empty_output':
self.__fail_if_empty_output = value
elif name == 'fail_only_if_all_failed':
if value in ['True', 'False']:
self.__fail_only_if_all_failed = eval(value)
elif name == 'split_by_comma':
if value in ['True', 'False']:
self.__split_by_comma = eval(value)
elif name == 'ha_dependant':
if value in ['True', 'False']:
self.__ha_dependant = eval(value)
elif name == 'version_type':
self.__version_type = value
elif name == 'severity':
if value in default.severity:
self.__severity = value
elif name == 'min_doctor_version':
self.__min_doctor_version = value
elif name == 'appliance_type':
for x in value.split(','):
self.__appliance_type += Wildcard.appliance_exec(
x.strip())
elif name == 'conditions':
self.__init_conditions__(value)
elif name == 'actions':
self.__init_actions__(value)
elif name == 'description':
self.__description = value
elif name == 'summary_passed':
self.__summary_passed = value
elif name == 'summary_failed':
self.__summary_failed = value
elif name == 'remediation':
self.__remediation = value
elif name == 'affects_strike_zone':
if value in ['True', 'False']:
self.__strike_zone = eval(value)
else:
Output.warning('Unknown field in check "%s": %s' %
(self.__name, name))
except CheckError:
raise
except Exception, msg:
Output.error('Cannot parse check "%s" in plugin "%s": %s' %
(self.__name, self.__plugin.get_name(), msg))
raise
def __init_conditions__(self, value):
# Check first if there are @set@ and other conditions in the same rule.
# This is not allowed because standalone data type checks rely on order,
# while @set tries to match with every field of the resulting regex/db query
# regardless the order.
if ('@set@' in value) and \
('@int@' in value or '@float@' in value or '@string@' in value or '@char@' in value or '@ipaddr@' in value):
raise CheckError(
'Forbidden "@set@" and any other datatype combination in rule "%s" for plugin "%s"'
% (self.__name, self.__plugin.get_name()), self.__name)
conditions = filter(bool, value.split(';'))
for condition in conditions:
matches = re.findall(r'^(@[a-zA-Z_]+@)(?:\:(.*))?$', condition)
if matches == []:
raise CheckError(
'Condition "%s" for check "%s" in plugin "%s" is invalid' %
(condition, self.__name, self.__plugin.get_name()),
self.__name)
cond_type, cond_str = matches[0]
# 'Basic' type conditions
if cond_type in [
'@string@', '@char@', '@int@', '@float@', '@info@',
'@ipaddr@'
]:
# Translate first, append later.
if cond_type in ['@ipaddr@']:
# Do not encapsulate in quotes, as this is an object comparison.
cond_str = Wildcard.av_config(cond_str,
encapsulate_str=False)
cond_str = Wildcard.ipaddr_operation(cond_str)
else:
key = re.findall(r'.*(@[a-zA-Z_]+@).*', cond_str)
cond_str = Wildcard.av_config(cond_str,
encapsulate_str=True)
if key:
self.__aux_data[key[0]] = Wildcard.av_config(
key[0], encapsulate_str=False)
self.__formatted_output = self.__formatted_output.replace(
key[0], self.__aux_data[key[0]])
self.__conditions['basic'].append(
(cond_type, cond_str.rsplit('@')
if cond_str != None and cond_str != '' else None))
# 'Set' type conditions
elif cond_type == '@set@':
matches = re.findall(r'^(@[a-zA-Z_]+@)(\S+)', cond_str)
if matches == []:
raise CheckError(
'Set condition "%s" for check "%s" in plugin "%s" is invalid'
% (condition, self.__name, self.__plugin.get_name()),
self.__name)
cond_op, cond_set = matches[0]
key = re.findall(r'.*(@[a-zA-Z_]+@).*', cond_set)
cond_set = Wildcard.av_config(cond_set)
if key:
self.__aux_data[key[0]] = Wildcard.av_config(
key[0], encapsulate_str=False)
self.__formatted_output = self.__formatted_output.replace(
key[0], self.__aux_data[key[0]])
if path.isfile(cond_set):
# For sets defined in files.
desc = open(cond_set, 'r')
items = desc.read().splitlines()
else:
items = cond_set.split(',')
content = set()
items = filter(None, items)
for item in items:
splitted_item = item.split('|')
if len(splitted_item) > 1:
content.add(tuple(splitted_item))
else:
content.add(item)
self.__conditions['set'].append(cond_op + str(content))
else:
raise CheckError(
'Type "%s" not recognized for check "%s" in plugin "%s"' %
(cond_type, self.__name, self.__plugin.get_name()),
self.__name)
def __init__(self, plugin, section):
# 'check' properties.
self.__name = ''
self.__type = ''
self.__pattern = ''
self.__category = ''
self.__description = ''
self.__summary_passed = ''
self.__summary_failed = ''
self.__remediation = ''
self.__plugin = None
# 'file' type checks only.
self.__checksums = []
# 'file' and 'command' checks.
self.__regex = None
# 'db' type checks.
self.__query = ''
self.__pivot = False
self.__introduced = ''
self.__output = ''
self.__formatted_output = ''
self.__appliance_type = []
self.__fail_if_empty = True
self.__fail_if_empty_output = ''
self.__fail_only_if_all_failed = False
self.__split_by_comma = False
self.__ha_dependant = False
self.__severity = 'Warning'
self.__conditions = {'basic': [], 'set': []}
self.__actions = []
self.__aux_data = {}
self.__strike_zone = False
self.__version_type = ''
config_file = plugin.get_config_file()
self.__name = section
self.__plugin = plugin
# Parse section options.
# Different sections or check 'types' are mutually exclusive.
items = config_file.items(section)
try:
# Retrieve first the formatted_output field
for (name, value) in items:
if name == 'formatted_output':
self.__formatted_output = value.replace("{nl}", "\n")
items.remove((name, value))
break
# Now the rest
for (name, value) in items:
if name == 'checksum':
self.__type = name
self.__checksums = [tuple(x.split(':')) for x in value.split(';')]
elif name == 'pattern':
self.__type = name
self.__pattern = str(value)
value = Wildcard.av_config(value, escape=True)
self.__regex = re.compile(value, re.MULTILINE)
elif name == 'query':
self.__type = name
if value.startswith("@pivot@:"):
self.__query = value[8:]
self.__pivot = True
else:
self.__query = value
self.__query = Wildcard.av_config(self.__query, escape=True)
elif name == 'hardware':
self.__type = name
self.__hw_list = value
elif name == 'category':
self.__category = value
elif name == 'fail_if_empty':
if value in ['True', 'False']:
self.__fail_if_empty = eval(value)
elif name == 'fail_if_empty_output':
self.__fail_if_empty_output = value
elif name == 'fail_only_if_all_failed':
if value in ['True', 'False']:
self.__fail_only_if_all_failed = eval(value)
elif name == 'split_by_comma':
if value in ['True', 'False']:
self.__split_by_comma = eval(value)
elif name == 'ha_dependant':
if value in ['True', 'False']:
self.__ha_dependant = eval(value)
elif name == 'version_type':
self.__version_type = value
elif name == 'severity':
if value in default.severity:
self.__severity = value
elif name == 'min_doctor_version':
self.__min_doctor_version = value
elif name == 'appliance_type':
for x in value.split(','):
self.__appliance_type += Wildcard.appliance_exec(x.strip())
elif name == 'conditions':
self.__init_conditions__(value)
elif name == 'actions':
self.__init_actions__(value)
elif name == 'description':
self.__description = value
elif name == 'summary_passed':
self.__summary_passed = value
elif name == 'summary_failed':
self.__summary_failed = value
elif name == 'remediation':
self.__remediation = value
elif name == 'affects_strike_zone':
if value in ['True', 'False']:
self.__strike_zone = eval(value)
else:
Output.warning('Unknown field in check "%s": %s' % (self.__name, name))
except CheckError:
raise
except Exception, msg:
Output.error('Cannot parse check "%s" in plugin "%s": %s' % (self.__name, self.__plugin.get_name(), msg))
raise
def __init_conditions__(self, value):
# Check first if there are @set@ and other conditions in the same rule.
# This is not allowed because standalone data type checks rely on order,
# while @set tries to match with every field of the resulting regex/db query
# regardless the order.
if ('@set@' in value) and \
('@int@' in value or '@float@' in value or '@string@' in value or '@char@' in value or '@ipaddr@' in value):
raise CheckError('Forbidden "@set@" and any other datatype combination in rule "%s" for plugin "%s"' % (self.__name, self.__plugin.get_name()), self.__name)
conditions = filter(bool, value.split(';'))
for condition in conditions:
matches = re.findall(r'^(@[a-zA-Z_]+@)(?:\:(.*))?$', condition)
if matches == []:
raise CheckError('Condition "%s" for check "%s" in plugin "%s" is invalid' % (condition, self.__name, self.__plugin.get_name()), self.__name)
cond_type, cond_str = matches[0]
# 'Basic' type conditions
if cond_type in ['@string@', '@char@', '@int@', '@float@', '@info@', '@ipaddr@']:
# Translate first, append later.
if cond_type in ['@ipaddr@']:
# Do not encapsulate in quotes, as this is an object comparison.
cond_str = Wildcard.av_config(cond_str, encapsulate_str=False)
cond_str = Wildcard.ipaddr_operation(cond_str)
else:
key = re.findall(r'.*(@[a-zA-Z_]+@).*', cond_str)
cond_str = Wildcard.av_config(cond_str, encapsulate_str=True)
if key:
self.__aux_data[key[0]] = Wildcard.av_config(key[0], encapsulate_str=False)
self.__formatted_output = self.__formatted_output.replace(key[0],
self.__aux_data[key[0]])
self.__conditions['basic'].append((cond_type, cond_str.rsplit('@') if cond_str != None and cond_str != '' else None))
# 'Set' type conditions
elif cond_type == '@set@':
matches = re.findall(r'^(@[a-zA-Z_]+@)(\S+)', cond_str)
if matches == []:
raise CheckError('Set condition "%s" for check "%s" in plugin "%s" is invalid' % (condition, self.__name, self.__plugin.get_name()), self.__name)
cond_op, cond_set = matches[0]
key = re.findall(r'.*(@[a-zA-Z_]+@).*', cond_set)
cond_set = Wildcard.av_config(cond_set)
if key:
self.__aux_data[key[0]] = Wildcard.av_config(key[0], encapsulate_str=False)
self.__formatted_output = self.__formatted_output.replace(key[0],
self.__aux_data[key[0]])
if path.isfile(cond_set):
# For sets defined in files.
desc = open(cond_set, 'r')
items = desc.read().splitlines()
else:
items = cond_set.split(',')
content = set()
items = filter(None, items)
for item in items:
splitted_item = item.split('|')
if len(splitted_item) > 1:
content.add(tuple(splitted_item))
else:
content.add(item)
self.__conditions['set'].append(cond_op + str(content))
else:
raise CheckError('Type "%s" not recognized for check "%s" in plugin "%s"' % (cond_type, self.__name, self.__plugin.get_name()), self.__name)