def check_files():
print("Note: This tool is for testing filters and config syntax. It will not process data or alert.\n")
parser = argparse.ArgumentParser(description='Validate a rule configuration')
parser.add_argument('files', metavar='file', type=str, nargs='+', help='rule configuration filename')
parser.add_argument('--schema-only', action='store_true', help='Show only schema errors; do not run query')
parser.add_argument('--days', type=int, default=[1, 7], nargs='+', help='Query the previous N days with this rule')
args = parser.parse_args()
for filename in args.files:
with open(filename) as fh:
conf = yaml.load(fh)
load_options(conf)
print("Successfully loaded %s\n" % (conf['name']))
if args.schema_only:
continue
es_client = Elasticsearch(host=conf['es_host'], port=conf['es_port'])
for days in args.days:
start_time = ts_now() - datetime.timedelta(days=days)
end_time = ts_now()
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(conf['filter'], starttime=start_time, endtime=end_time, timestamp_field=ts)
index = ElastAlerter.get_index(conf, start_time, end_time)
try:
res = es_client.search(index, size=1000, body=query)
except Exception as e:
print("Error running your filter:")
print(repr(e)[:2048])
exit(1)
num_hits = len(res['hits']['hits'])
print("Got %s hits from the last %s day%s" % (num_hits if num_hits != 1000 else '1000+', days,
's' if days > 1 else ''))
if num_hits:
print("\nAvailable terms in first hit:")
terms = res['hits']['hits'][0]['_source']
print_terms(terms, '')
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!")
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!")
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term))
for term in conf.get('top_count_keys', []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')):
print("top_count_key %s may be missing" % (term))
print('')
def check_files():
print("Note: This tool is for testing filters and config syntax. It will not process data or alert.\n")
parser = argparse.ArgumentParser(description='Validate a rule configuration')
parser.add_argument('files', metavar='file', type=str, nargs='+', help='rule configuration filename')
parser.add_argument('--days', type=int, default=[1, 7], nargs='+', help='Query the previous N days with this rule')
args = parser.parse_args()
for filename in args.files:
with open(filename) as fh:
conf = yaml.load(fh)
load_options(conf)
print("Successfully loaded %s\n" % (conf['name']))
es_client = Elasticsearch(host=conf['es_host'], port=conf['es_port'])
for days in args.days:
start_time = ts_now() - datetime.timedelta(days=days)
end_time = ts_now()
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(conf['filter'], starttime=start_time, endtime=end_time, timestamp_field=ts)
index = ElastAlerter.get_index(conf, start_time, end_time)
try:
res = es_client.search(index, size=1000, body=query)
except Exception as e:
print("Error running your filter:")
print(repr(e)[:2048])
exit(1)
num_hits = len(res['hits']['hits'])
print("Got %s hits from the last %s day%s" % (num_hits if num_hits != 1000 else '1000+', days,
's' if days > 1 else ''))
if num_hits:
print("\nAvailable terms in first hit:")
terms = res['hits']['hits'][0]['_source']
print_terms(terms, '')
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!")
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!")
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term))
for term in conf.get('top_count_keys', []):
if term not in terms:
print("top_count_key %s may be missing" % (term))
print('')
def test_file(self, conf, args):
""" Loads a rule config file, performs a query over the last day (args.days), lists available keys
and prints the number of results. """
if args.schema_only:
return []
# Set up Elasticsearch client and query
es_client = elasticsearch_client(conf)
try:
is_five = es_client.info()['version']['number'].startswith('5')
except Exception as e:
print("Error connecting to ElasticSearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
if is_five:
ElastAlerter.modify_rule_for_ES5(conf)
start_time = ts_now() - datetime.timedelta(days=args.days)
end_time = ts_now()
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(conf['filter'],
starttime=start_time,
endtime=end_time,
timestamp_field=ts,
five=is_five)
print('test query: ' + str(query))
index = ElastAlerter.get_index(conf, start_time, end_time)
# Get one document for schema
try:
res = es_client.search(index,
size=1,
body=query,
ignore_unavailable=True)
print('test res: ' + str(res))
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = len(res['hits']['hits'])
if not num_hits:
return []
terms = res['hits']['hits'][0]['_source']
doc_type = res['hits']['hits'][0]['_type']
# Get a count of all docs
count_query = ElastAlerter.get_query(conf['filter'],
starttime=start_time,
endtime=end_time,
timestamp_field=ts,
sort=False,
five=is_five)
try:
res = es_client.count(index,
doc_type=doc_type,
body=count_query,
ignore_unavailable=True)
except Exception as e:
print("Error querying Elasticsearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = res['count']
print("Got %s hits from the last %s day%s" %
(num_hits, args.days, 's' if args.days > 1 else ''))
print("\nAvailable terms in first hit:")
print_terms(terms, '')
# Check for missing keys
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!",
file=sys.stderr)
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!",
file=sys.stderr)
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term),
file=sys.stderr)
for term in conf.get('top_count_keys', []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (term.endswith('.raw')
and term[:-4] in terms
and index.startswith('logstash')):
print("top_count_key %s may be missing" % (term),
file=sys.stderr)
print('') # Newline
# Download up to 10,000 documents to save
if args.save and not args.count:
try:
res = es_client.search(index,
size=10000,
body=query,
ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = len(res['hits']['hits'])
print("Downloaded %s documents to save" % (num_hits))
return res['hits']['hits']
def test_file(self, conf, args):
""" Loads a rule config file, performs a query over the last day (args.days), lists available keys
and prints the number of results. """
load_options(conf, {})
print("Successfully loaded %s\n" % (conf['name']))
if args.schema_only:
return []
# Set up elasticsearch client and query
es_config = ElastAlerter.build_es_conn_config(conf)
es_client = ElastAlerter.new_elasticsearch(es_config)
start_time = ts_now() - datetime.timedelta(days=args.days)
end_time = ts_now()
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(conf['filter'], starttime=start_time, endtime=end_time, timestamp_field=ts)
index = ElastAlerter.get_index(conf, start_time, end_time)
# Get one document for schema
try:
res = es_client.search(index, size=1, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = len(res['hits']['hits'])
if not num_hits:
return []
terms = res['hits']['hits'][0]['_source']
doc_type = res['hits']['hits'][0]['_type']
# Get a count of all docs
count_query = ElastAlerter.get_query(conf['filter'], starttime=start_time, endtime=end_time, timestamp_field=ts, sort=False)
count_query = {'query': {'filtered': count_query}}
try:
res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True)
except Exception as e:
print("Error querying Elasticsearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = res['count']
print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else ''))
print("\nAvailable terms in first hit:")
print_terms(terms, '')
# Check for missing keys
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!", file=sys.stderr)
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!", file=sys.stderr)
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term), file=sys.stderr)
for term in conf.get('top_count_keys', []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')):
print("top_count_key %s may be missing" % (term), file=sys.stderr)
print('') # Newline
# Download up to 10,000 documents to save
if args.save and not args.count:
try:
res = es_client.search(index, size=10000, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = len(res['hits']['hits'])
print("Downloaded %s documents to save" % (num_hits))
return res['hits']['hits']
return None
def test_file(self, conf):
"""Loads a rule config file, performs a query over the last day (self.args.days), lists available keys
and prints the number of results."""
if self.args.schema_only:
return []
# Set up Elasticsearch client and query
es_client = elasticsearch_client(conf)
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(conf['filter'],
starttime=self.starttime,
endtime=self.endtime,
timestamp_field=ts,
to_ts_func=conf['dt_to_ts'])
index = ElastAlerter.get_index(conf, self.starttime, self.endtime)
# Get one document for schema
try:
res = es_client.search(index=index,
size=1,
body=query,
ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if self.args.stop_error:
exit(3)
return None
num_hits = len(res['hits']['hits'])
if not num_hits:
print("Didn't get any results.")
return []
terms = res['hits']['hits'][0]['_source']
# Get a count of all docs
count_query = ElastAlerter.get_query(conf['filter'],
starttime=self.starttime,
endtime=self.endtime,
timestamp_field=ts,
to_ts_func=conf['dt_to_ts'],
sort=False)
try:
res = es_client.count(index=index,
body=count_query,
ignore_unavailable=True)
except Exception as e:
print("Error querying Elasticsearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if self.args.stop_error:
exit(2)
return None
num_hits = res['count']
if self.args.formatted_output:
self.formatted_output['hits'] = num_hits
self.formatted_output['days'] = self.args.days
self.formatted_output['terms'] = list(terms.keys())
self.formatted_output['result'] = terms
else:
print(
"Got %s hits from the last %s day%s" %
(num_hits, self.args.days, "s" if self.args.days > 1 else ""))
print("\nAvailable terms in first hit:")
print_terms(terms, '')
# Check for missing keys
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!",
file=sys.stderr)
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!",
file=sys.stderr)
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term),
file=sys.stderr)
for term in conf.get('top_count_keys', []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (term.endswith('.raw')
and term[:-4] in terms
and index.startswith('logstash')):
print("top_count_key %s may be missing" % (term),
file=sys.stderr)
if not self.args.formatted_output:
print('') # Newline
# Download up to max_query_size (defaults to 10,000) documents to save
if (self.args.save
or self.args.formatted_output) and not self.args.count:
try:
res = es_client.search(index=index,
size=self.args.max_query_size,
body=query,
ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if self.args.stop_error:
exit(2)
return None
num_hits = len(res['hits']['hits'])
if self.args.save:
print("Downloaded %s documents to save" % (num_hits))
return res['hits']['hits']
def test_file(self, conf, args):
""" Loads a rule config file, performs a query over the last day (args.days), lists available keys
and prints the number of results. """
if args.schema_only:
return []
# Set up Elasticsearch client and query
es_client = elasticsearch_client(conf)
try:
ElastAlerter.modify_rule_for_ES5(conf)
except Exception as e:
print("Error connecting to ElasticSearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
start_time = ts_now() - datetime.timedelta(days=args.days)
end_time = ts_now()
ts = conf.get('timestamp_field', '@timestamp')
query = ElastAlerter.get_query(
conf['filter'],
starttime=start_time,
endtime=end_time,
timestamp_field=ts,
to_ts_func=conf['dt_to_ts'],
five=conf['five']
)
index = ElastAlerter.get_index(conf, start_time, end_time)
# Get one document for schema
try:
res = es_client.search(index, size=1, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = len(res['hits']['hits'])
if not num_hits:
return []
terms = res['hits']['hits'][0]['_source']
doc_type = res['hits']['hits'][0]['_type']
# Get a count of all docs
count_query = ElastAlerter.get_query(
conf['filter'],
starttime=start_time,
endtime=end_time,
timestamp_field=ts,
to_ts_func=conf['dt_to_ts'],
sort=False,
five=conf['five']
)
try:
res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True)
except Exception as e:
print("Error querying Elasticsearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = res['count']
if args.formatted_output:
self.formatted_output['hits'] = num_hits
self.formatted_output['days'] = args.days
self.formatted_output['terms'] = terms.keys()
self.formatted_output['result'] = terms
else:
print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else ''))
print("\nAvailable terms in first hit:")
print_terms(terms, '')
# Check for missing keys
pk = conf.get('primary_key')
ck = conf.get('compare_key')
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!", file=sys.stderr)
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!", file=sys.stderr)
include = conf.get('include')
if include:
for term in include:
if not lookup_es_key(terms, term) and '*' not in term:
print("Included term %s may be missing or null" % (term), file=sys.stderr)
for term in conf.get('top_count_keys', []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')):
print("top_count_key %s may be missing" % (term), file=sys.stderr)
if not args.formatted_output:
print('') # Newline
# Download up to max_query_size (defaults to 10,000) documents to save
if (args.save or args.formatted_output) and not args.count:
try:
res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
if args.stop_error:
exit(1)
return None
num_hits = len(res['hits']['hits'])
if args.save:
print("Downloaded %s documents to save" % (num_hits))
return res['hits']['hits']
def test_file(self, args):
""" Loads a rule config file, performs a query over the last day (args.days), lists available keys
and prints the number of results. """
filename = args.file
with open(filename) as fh:
conf = yaml.load(fh)
load_options(conf)
print("Successfully loaded %s\n" % (conf["name"]))
if args.schema_only:
return []
# Set up elasticsearch client and query
es_client = Elasticsearch(host=conf["es_host"], port=conf["es_port"])
start_time = ts_now() - datetime.timedelta(days=args.days)
end_time = ts_now()
ts = conf.get("timestamp_field", "@timestamp")
query = ElastAlerter.get_query(conf["filter"], starttime=start_time, endtime=end_time, timestamp_field=ts)
index = ElastAlerter.get_index(conf, start_time, end_time)
# Get one document for schema
try:
res = es_client.search(index, size=1, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = len(res["hits"]["hits"])
if not num_hits:
return []
terms = res["hits"]["hits"][0]["_source"]
doc_type = res["hits"]["hits"][0]["_type"]
# Get a count of all docs
count_query = ElastAlerter.get_query(
conf["filter"], starttime=start_time, endtime=end_time, timestamp_field=ts, sort=False
)
count_query = {"query": {"filtered": count_query}}
try:
res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True)
except Exception as e:
print("Error querying Elasticsearch:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = res["count"]
print("Got %s hits from the last %s day%s" % (num_hits, args.days, "s" if args.days > 1 else ""))
print("\nAvailable terms in first hit:")
print_terms(terms, "")
# Check for missing keys
pk = conf.get("primary_key")
ck = conf.get("compare_key")
if pk and not lookup_es_key(terms, pk):
print("Warning: primary key %s is either missing or null!", file=sys.stderr)
if ck and not lookup_es_key(terms, ck):
print("Warning: compare key %s is either missing or null!", file=sys.stderr)
include = conf.get("include")
if include:
for term in include:
if not lookup_es_key(terms, term) and "*" not in term:
print("Included term %s may be missing or null" % (term), file=sys.stderr)
for term in conf.get("top_count_keys", []):
# If the index starts with 'logstash', fields with .raw will be available but won't in _source
if term not in terms and not (
term.endswith(".raw") and term[:-4] in terms and index.startswith("logstash")
):
print("top_count_key %s may be missing" % (term), file=sys.stderr)
print("") # Newline
# Download up to 10,000 documents to save
if args.save and not args.count:
try:
res = es_client.search(index, size=10000, body=query, ignore_unavailable=True)
except Exception as e:
print("Error running your filter:", file=sys.stderr)
print(repr(e)[:2048], file=sys.stderr)
return None
num_hits = len(res["hits"]["hits"])
print("Downloaded %s documents to save" % (num_hits))
return res["hits"]["hits"]
return None