def setUp(self): """ Initialize the system configuration and the user configuration. Note the forward slash replacements for the user configuration. This is due to the forward slash being a restricted character in TOML(package used to parse configuration files in LQMT). """ # relative pathing variables. Replace function calls for Windows compatibility. self.directory = os.path.dirname(__file__) self.alerts = self.directory + "/test_data/" self.alerts = self.alerts.replace("\\", "/") self.logging = self.directory + "/test_data/test-logs/lqmt" self.logging = self.logging.replace("\\", "/") self.whitelist = self.directory + "/test_data/whitelist/whitelist.txt" self.whitelist = self.whitelist.replace("\\", "/") self.whitelist_db = self.directory + "/test_data/whitelist/whitelist.db" self.whitelist_db = self.whitelist_db.replace("\\", "/") # configurations initialized sysconf = SystemConfig() self.sys_config = sysconf.getConfig() config = USERCONFIG.format(self.alerts, self.logging, self.whitelist, self.whitelist_db) self.toml_config = toml.loads(config) self.toml_config = self.toml_config["Tools"]["FlexText"][0] # dirty way of parsing userconfig for ToolConfig self.user_config = LQMToolConfig(config) self.toolConfig = ToolConfig(self.toml_config, csvToolInfo={""}, unhandledCSV={""})
def setUp(self): sysconf = SystemConfig() self.sys_config = sysconf.getConfig() self.sys_config = self.sys_config['parsers'] self.stixparser = STIXParser() self.time = str(time.time()).split('.')[0] self.directory = os.path.dirname(__file__) self.test_file = self.directory + "/" + STIXRULES
def setUp(self): sysconf = SystemConfig() self.sys_config = sysconf.getConfig() self.sys_config = self.sys_config['parsers'] self.ruleparser = RuleParser() self.time = str(time.time()).split('.')[0] self.directory = os.path.dirname(__file__) self.test_file = self.directory + "/" + SNORTEXAMPLE self.yara_test_file = self.directory + "/" + YARAFILE
def setUp(self): sysconf = SystemConfig() self.sys_config = sysconf.getConfig() self.sys_config = self.sys_config['parsers'] self.flext = FlexTransformParser() # Add parsers for the different data formats self.flext.add_parser('LQMTools', 'resources/sampleConfigurations/lqmtools.cfg') self.flext.add_parser('Cfm13Alert', 'resources/sampleConfigurations/cfm13.cfg') self.flext.add_parser('Cfm20Alert', 'resources/sampleConfigurations/cfm20alert.cfg') self.flext.add_parser('stixtlp', 'resources/sampleConfigurations/stix_tlp.cfg') self.flext.add_parser('STIX', 'resources/sampleConfigurations/stix_tlp.cfg') # Standard timestamp that can be used so all meta files use the same time and for easier validation self.time = str(time.time()).split('.')[0] # Parse all the data ahead of the tests self.cfm13_parsed_data = self.flext.parse(io.StringIO(CFM13ALERT), self.getMeta("Cfm13Alert")) self.cfm20_parsed_data = self.flext.parse(io.StringIO(CFM20ALERT), self.getMeta("Cfm20Alert")) self.stix_parsed_data = self.flext.parse(io.StringIO(STIX), self.getMeta("STIX"))
def _loadSystemConfig(self): """Load the System configuration file""" sysconf = SystemConfig() self._config = sysconf.getConfig()
class FlexTextConfig(ToolConfig): """ Configuration Class for FlexText """ def __init__(self, config_data, csvToolInfo, unhandledCSV): """ Constructor """ super().__init__(config_data, csvToolInfo, unhandledCSV) self.logger = logging.getLogger("LQMT.FlexText.{0}".format(self.getName())) # FlexTransform configuration variables self.header_line = False self.increment_file = False self.flext_config = 'resources/sampleConfigurations/flextext.cfg' self.system_config = SystemConfig() self.system_config = self.system_config.getConfig() self.config_dict = {} self.config_str = "" # Configure FlexT parsers based on system configuration self.source_configs = {} for parser_name, parsers in self.system_config['parsers'].items(): # Exceptions: LQMTools is a part of every set of parser # del (parsers['configs']['LQMTools']) # TODO: needs testing, change because new parser if 'LQMTools' in parsers['configs']: del (parsers['configs']['LQMTools']) # If it's more specific, like MBL, then you need to first check that it exists first during iteration by # checking the parser_name if parser_name == 'mbl': del (parsers['configs']['mbl']) # Temporary fix for failing tests. Tests fail due to current FlexT version. Once FlexT version 1.2.1 is # released, this fix can be removed. ft_version = get_distribution('FlexTransform').version ft_version = tuple([int(x) for x in ft_version.split('.')]) if ft_version < (1, 2, 1): if "IID" in parser_name: del (parsers['configs'][parser_name]) # after the exceptions, update the source_configs with remaining parser configs self.source_configs.update(parsers['configs']) # FlexText configuration variables self.fileParser = self.validation('fileParser', str, default="CSV") self.delimiter = self.validation('delimiter', str, required=True) self.quote_char = self.validation('quoteChar', str, required=True) self.escape_char = self.validation('escapeChar', str, required=True) self.header_line = self.validation('headerLine', bool, default=True) self.double_quote = self.validation('doubleQuote', bool) self.quote_style = self.validation('quoteStyle', str, default="none") self.primary_schema_config = self.validation('primarySchemaConfig', str, default="resources/schemaDefinitions" "/lqmtools.json") self.increment_file = self.validation('incrementFile', bool) self.file_destination = self.validation('fileDestination', str, required=True) # fields variable is configured differently for legacy purposes. The fields variable was originally a string, # so we are first checking to see if it's a string (legacy configs), if so we convert it to a list. Then we # validate it. if 'fields' in config_data: if isinstance(config_data['fields'], str): config_data['fields'] = config_data['fields'].split(',') self.fields = self.validation('fields', list, required=True) if self.increment_file: base, extension = os.path.splitext(self.file_destination) file_name = "." + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") self.file_destination = base + file_name + extension def config_to_dict(self): """ Takes config vars and turns them into a FlexT compatible config dict; assigns values to config_dict :return: returns config_dict var """ self.config_dict = { 'SYNTAX': { 'FileParser': self.fileParser }, 'CSV': { 'Fields': ','.join(self.fields), 'Delimiter': self.delimiter, 'QuoteChar': self.quote_char, 'EscapeChar': self.escape_char, 'HeaderLine': self.header_line, 'DoubleQuote': self.double_quote, 'QuoteStyle': self.quote_style }, 'SCHEMA': { 'PrimarySchemaConfiguration': self.primary_schema_config } } return self.config_dict def config_to_str(self): """ Takes config vars and turns them into a FlexT compatible config string; assigns values to config_str :return: returns config_str var """ self.config_str += "[SYNTAX]" self.config_str += "\nFileParser=" + self.fileParser self.config_str += "\n[CSV]" self.config_str += "\nFields=" + ','.join(self.fields) self.config_str += "\nDelimiter='" + self.delimiter + "'" self.config_str += "\nQuoteChar=" + self.quote_char self.config_str += "\nEscapeChar=" + self.escape_char self.config_str += "\nHeaderLine=" + str(self.header_line) self.config_str += "\nDoubleQuote=" + str(self.double_quote) self.config_str += "\nQuoteStyle=" + str(self.quote_style) self.config_str += "\n[SCHEMA]" self.config_str += "\nPrimarySchemaConfiguration=" + self.primary_schema_config return self.config_str