def clean(self, value):
cleaned_value = super(PatternProperty, self).clean(value)
errors = run_validator(cleaned_value)
if errors:
raise ValueError(str(errors[0]))
return cleaned_value
def clean(self, value):
str_value = super(PatternProperty, self).clean(value)
errors = run_validator(str_value)
if errors:
raise ValueError(str(errors[0]))
return self.string_type(value)
def _validate_pattern(self, pattern):
# Validator doesn't support START STOP qualifier so strip out before validating pattern
start_stop_pattern = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?"
pattern_without_start_stop = re.sub(start_stop_pattern, " ", pattern)
errors = run_validator(pattern_without_start_stop)
if (errors != []):
raise StixValidationException("The STIX pattern has the following errors: {}".format(errors))
def main():
if len(sys.argv) <= 2:
return_data(
{"status": "error", "message": "Missing argument to the Python script"}
)
if sys.argv[1] == "check":
return_data({"status": "success"})
pattern_type = sys.argv[1]
indicator_value = sys.argv[2]
if pattern_type == "stix":
result = False
try:
errors = run_validator(indicator_value)
if len(errors) == 0:
result = True
except:
result = False
return_data({"status": "success", "data": result})
if pattern_type == "yara":
parser = plyara.Plyara()
result = False
try:
parser.parse_string(indicator_value)
result = True
except:
result = False
return_data({"status": "success", "data": result})
if pattern_type == "sigma":
result = False
try:
parser = SigmaCollectionParser(indicator_value)
result = True
except:
result = False
return_data({"status": "success", "data": result})
if pattern_type == "snort":
result = False
try:
parsed = Parser(indicator_value).all
result = True
except:
result = False
return_data({"status": "success", "data": result})
if pattern_type == "suricata":
result = False
try:
parsed = parse_rules(indicator_value)
result = True
except:
result = False
return_data({"status": "success", "data": result})
return_data({"status": "unknown", "data": None})
def _validate_pattern(self, pattern):
# Validator doesn't support START STOP qualifier so strip out before validating pattern
pattern_without_start_stop = re.sub(START_STOP_PATTERN, " ", pattern)
errors = run_validator(pattern_without_start_stop)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(errors))
def main():
if len(sys.argv) <= 2:
return_data({'status': 'error', 'message': 'Missing argument to the Python script'})
if sys.argv[1] == 'check':
return_data({'status': 'success'})
pattern_type = sys.argv[1]
indicator_value = sys.argv[2]
if pattern_type == 'stix':
result = False
try:
errors = run_validator(indicator_value)
if len(errors) == 0:
result = True
except:
result = False
return_data({'status': 'success', 'data': result})
if pattern_type == 'yara':
parser = plyara.Plyara()
result = False
try:
parser.parse_string(indicator_value)
result = True
except:
result = False
return_data({'status': 'success', 'data': result})
if pattern_type == 'sigma':
result = False
try:
parser = SigmaCollectionParser(indicator_value)
result = True
except:
result = False
return_data({'status': 'success', 'data': result})
if pattern_type == 'snort':
result = False
try:
parsed = Parser(indicator_value).all
result = True
except:
result = False
return_data({'status': 'success', 'data': result})
if pattern_type == 'suricata':
result = False
try:
parsed = parse_rules(indicator_value)
result = True
except:
result = False
return_data({'status': 'success', 'data': result})
return_data({'status': 'unknown', 'data': None})
def _validate_pattern(self, pattern):
errors = []
# Temporary work around since pattern validator currently treats multiple qualifiers of the same type as invalid.
start_stop_count = len(re.findall(START_STOP_PATTERN, pattern))
if(start_stop_count > 1):
pattern = re.sub(START_STOP_PATTERN, " ", pattern)
errors = run_validator(pattern, stix_version='2.1')
if (errors):
raise StixValidationException("The STIX pattern has the following errors: {}".format(errors))
def translate(self, module, translate_type, data_source, data, options={}):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:return: translated results
:rtype: str
"""
dialect = None
mod_dia = module.split(':', 1)
module = mod_dia[0]
if len(mod_dia) > 1:
dialect = mod_dia[1]
if module not in TRANSLATION_MODULES:
raise NotImplementedError
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." +
module + "_translator")
if dialect is not None:
interface = translator_module.Translator(dialect=dialect)
else:
interface = translator_module.Translator()
if translate_type == QUERY:
errors = []
# Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format
start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP"
pattern_match = bool(re.search(start_stop_pattern, data))
if (not pattern_match):
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(
errors))
else:
# Translating STIX pattern to antlr query object
query_object = generate_query(data)
# Converting query object to datasource query
parsed_stix = parse_stix(query_object)
# Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls.
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, options)
return {'queries': queries, 'parsed_stix': parsed_stix}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return interface.translate_results(data_source, data, options)
else:
raise NotImplementedError
def transform_query(self, data, options, mapping=None):
"""
Transforms STIX query into a different query format. based on a mapping file
:param data: STIX query string to transform into another format
:type data: str
:param mapping: The mapping file path to use as instructions on how to transform the given STIX query into another format. This should default to something if one isn't passed in
:type mapping: str (filepath)
:return: transformed query string
:rtype: str
"""
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(errors))
else:
return self.query_translator.transform_query(data, options, mapping)
def _check_object_constraints(self):
super(Indicator, self)._check_object_constraints()
valid_from = self.get('valid_from')
valid_until = self.get('valid_until')
if valid_from and valid_until and valid_until <= valid_from:
msg = "{0.id} 'valid_until' must be greater than 'valid_from'"
raise ValueError(msg.format(self))
if self.get('pattern_type') == "stix":
try:
pat_ver = self.get('pattern_version')
except AttributeError:
pat_ver = '2.1'
errors = run_validator(self.get('pattern'), pat_ver)
if errors:
raise InvalidValueError(self.__class__, 'pattern', str(errors[0]))
def translate(self, module, translate_type, data_source, data, options={}):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:return: translated results
:rtype: str
"""
if module not in MODULES:
raise NotImplementedError
translator_module = importlib.import_module(
"stix_shifter.src.modules." + module + "." + module + "_translator")
interface = translator_module.Translator()
if translate_type == QUERY:
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(errors))
else:
# Translating STIX pattern to antlr query object
query_object = generate_query(data)
# Converting query object to datasource query
parsed_stix = stix_pattern_parser.parse_stix(query_object)
# Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls.
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, options)
return {'queries': queries, 'parsed_stix': parsed_stix}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return interface.translate_results(data_source, data, options)
else:
raise NotImplementedError
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('stix2-pattern'):
misperrors['error'] = 'STIX2 pattern missing'
return misperrors
pattern = request.get('stix2-pattern')
syntax_errors = []
for p in pattern[2:-2].split(' AND '):
syntax_validator = run_validator("[{}]".format(p))
if syntax_validator:
for error in syntax_validator:
syntax_errors.append(error)
if syntax_errors:
s = 's' if len(syntax_errors) > 1 else ''
s_errors = ""
for error in syntax_errors:
s_errors += "{}\n".format(error[6:])
result = "Syntax error{}: \n{}".format(s, s_errors[:-1])
else:
result = "Syntax valid"
return {'results': [{'types': mispattributes['output'], 'values': result}]}
def validate_pattern(pb, oid, pattern):
# Additional logic could be added here
errors = run_validator(pattern)
for e in errors:
print '{} - {}\n{}\n{}\n'.format(pb, oid, pattern, e)
def translate(self,
module,
translate_type,
data_source,
data,
options={},
recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
dialect = None
mod_dia = module.split(':', 1)
module = mod_dia[0]
if len(mod_dia) > 1:
dialect = mod_dia[1]
try:
if module not in TRANSLATION_MODULES:
raise UnsupportedDataSourceException(
"{} is an unsupported data source.".format(module))
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." +
module + "_translator")
if dialect is not None:
interface = translator_module.Translator(dialect=dialect)
else:
interface = translator_module.Translator()
if translate_type == QUERY:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
print(
"Changing Python recursion limit from {} to {}".format(
current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
if 'result_limit' not in options:
options['result_limit'] = DEFAULT_LIMIT
if 'timerange' not in options:
options['timerange'] = DEFAULT_TIMERANGE
errors = []
# Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format
start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP"
pattern_match = re.search(start_stop_pattern, data)
if (not pattern_match):
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(
errors))
else:
# Translating STIX pattern to antlr query object
query_object = generate_query(data)
# Converting query object to datasource query
parsed_stix_dictionary = parse_stix(
query_object, options['timerange'])
parsed_stix = parsed_stix_dictionary['parsed_stix']
start_time = parsed_stix_dictionary['start_time']
end_time = parsed_stix_dictionary['end_time']
# Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls.
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, options)
return {
'queries': queries,
'parsed_stix': parsed_stix,
'start_time': start_time,
'end_time': end_time
}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
try:
return interface.translate_results(data_source, data,
options)
except Exception:
raise TranslationResultException()
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
print('caught exception: ' + str(ex) + " " + str(type(ex)))
response = dict()
ErrorResponder.fill_error(response,
message_struct={'exception': ex})
return response
def _check_object_constraints(self):
errors = run_validator(self.get('pattern'), '2.0')
if errors:
raise InvalidValueError(self.__class__, 'pattern', str(errors[0]))
