def create_query_pack_from_upload(upload):
'''
Create a pack and queries from a query pack file. **Note**, if a
pack already exists under the filename being uploaded, then any
queries defined here will be added to the existing pack! However,
if a query with a particular name already exists, and its sql is
NOT the same, then a new query with the same name but different id
will be created (as to avoid clobbering the existing query). If its
sql is identical, then the query will be reused.
'''
from doorman.models import Pack, Query
# The json package on Python 3 expects a `str` input, so we're going to
# read the body and possibly convert to the right type
body = upload.data.read()
if not isinstance(body, six.string_types):
body = body.decode('utf-8')
data = json.loads(body)
name = splitext(basename(upload.data.filename))[0]
pack = Pack.query.filter(Pack.name == name).first()
if not pack:
current_app.logger.debug("Creating pack %s", name)
pack = Pack.create(name=name, **data)
for query_name, query in data['queries'].items():
if not validate_osquery_query(query['query']):
flash('Invalid osquery query: "{0}"'.format(query['query']), 'danger')
return None
q = Query.query.filter(Query.name == query_name).first()
if not q:
q = Query.create(name=query_name, **query)
pack.queries.append(q)
current_app.logger.debug("Adding new query %s to pack %s",
q.name, pack.name)
continue
if q in pack.queries:
continue
if q.sql == query['query']:
current_app.logger.debug("Adding existing query %s to pack %s",
q.name, pack.name)
pack.queries.append(q)
else:
q2 = Query.create(name=query_name, **query)
current_app.logger.debug(
"Created another query named %s, but different sql: %r vs %r",
query_name, q2.sql.encode('utf-8'), q.sql.encode('utf-8'))
pack.queries.append(q2)
else:
pack.save()
flash(u"Imported query pack {0}".format(pack.name), 'success')
return pack
def test_valid_configuration(self, node, testapp):
resp = testapp.post_json(url_for('api.configuration'),
{'node_key': node.node_key})
assert resp.json['node_invalid'] is False
first_config = resp.json
first_config.pop('node_invalid')
assert first_config == node.get_config()
tag = Tag.create(value='foo')
node.tags.append(tag)
node.save()
# test adding a tag to a query results in the query being included
# for this configuration
query1 = Query.create(name='bar', sql='select * from osquery_info;')
query2 = Query.create(name='foobar', sql='select * from system_info;')
query2.tags.append(tag)
query2.save()
# test adding a tag to a pack results in the pack being included
# for this configuration
pack = Pack.create(name='baz')
pack.queries.append(query1)
pack.tags.append(tag)
pack.save()
file_path = FilePath.create(category='foobar',
target_paths=[
'/home/foobar/%%',
])
file_path.tags.append(tag)
file_path.save()
resp = testapp.post_json(url_for('api.configuration'),
{'node_key': node.node_key})
assert resp.json['node_invalid'] is False
second_config = resp.json
second_config.pop('node_invalid')
assert second_config != first_config
assert second_config == node.get_config()
def test_config(self):
node = NodeFactory(host_identifier='foo')
tag = Tag.create(value='foo')
node.tags.append(tag)
node.save()
query1 = Query.create(name='bar', sql='select * from osquery_info;')
query2 = Query.create(name='foobar', sql='select * from system_info;')
query2.tags.append(tag)
query2.save()
pack = Pack.create(name='baz')
pack.queries.append(query1)
pack.tags.append(tag)
pack.save()
file_path = FilePath.create(category='foobar',
target_paths=[
'/home/foobar/%%',
])
file_path.tags.append(tag)
file_path.save()
assert tag in pack.tags
assert tag in query2.tags
assert tag in file_path.tags
assert tag not in query1.tags
assert query1 in pack.queries
assert query2 not in pack.queries
assert pack in node.packs
assert query2 in node.queries
assert query1 not in node.queries
config = node.get_config()
assert node.host_identifier == config['options']['host_identifier']
assert pack.name in config['packs']
assert query1.name in config['packs'][pack.name]['queries']
assert query1.sql == config['packs'][pack.name]['queries'][
query1.name]['query']
assert query2.name not in config['packs']
assert query2.name in config['schedule']
assert query2.sql == config['schedule'][query2.name]['query']
assert file_path.category in config['file_paths']
def test_valid_configuration(self, node, testapp):
resp = testapp.post_json(url_for('api.configuration'), {
'node_key': node.node_key})
assert resp.json['node_invalid'] is False
first_config = resp.json
first_config.pop('node_invalid')
assert first_config == node.get_config()
tag = Tag.create(value='foo')
node.tags.append(tag)
node.save()
# test adding a tag to a query results in the query being included
# for this configuration
query1 = Query.create(name='bar', sql='select * from osquery_info;')
query2 = Query.create(name='foobar', sql='select * from system_info;')
query2.tags.append(tag)
query2.save()
# test adding a tag to a pack results in the pack being included
# for this configuration
pack = Pack.create(name='baz')
pack.queries.append(query1)
pack.tags.append(tag)
pack.save()
file_path = FilePath.create(category='foobar', target_paths=[
'/home/foobar/%%',
])
file_path.tags.append(tag)
file_path.save()
resp = testapp.post_json(url_for('api.configuration'), {
'node_key': node.node_key})
assert resp.json['node_invalid'] is False
second_config = resp.json
second_config.pop('node_invalid')
assert second_config != first_config
assert second_config == node.get_config()
def test_config(self):
node = NodeFactory(host_identifier='foo')
tag = Tag.create(value='foo')
node.tags.append(tag)
node.save()
query1 = Query.create(name='bar', sql='select * from osquery_info;')
query2 = Query.create(name='foobar', sql='select * from system_info;')
query2.tags.append(tag)
query2.save()
pack = Pack.create(name='baz')
pack.queries.append(query1)
pack.tags.append(tag)
pack.save()
file_path = FilePath.create(category='foobar', target_paths=[
'/home/foobar/%%',
])
file_path.tags.append(tag)
file_path.save()
assert tag in pack.tags
assert tag in query2.tags
assert tag in file_path.tags
assert tag not in query1.tags
assert query1 in pack.queries
assert query2 not in pack.queries
assert pack in node.packs
assert query2 in node.queries
assert query1 not in node.queries
config = node.get_config()
assert node.host_identifier == config['options']['host_identifier']
assert pack.name in config['packs']
assert query1.name in config['packs'][pack.name]['queries']
assert query1.sql == config['packs'][pack.name]['queries'][query1.name]['query']
assert query2.name not in config['packs']
assert query2.name in config['schedule']
assert query2.sql == config['schedule'][query2.name]['query']
assert file_path.category in config['file_paths']
def add_query():
form = CreateQueryForm()
form.set_choices()
if form.validate_on_submit():
query = Query(name=form.name.data,
sql=form.sql.data,
interval=form.interval.data,
platform=form.platform.data,
version=form.version.data,
description=form.description.data,
value=form.value.data)
query.tags = create_tags(*form.tags.data.splitlines())
query.save()
return redirect(url_for('.query', query_id=query.id))
flash_errors(form)
return render_template('query.html', form=form)
def add_query():
form = CreateQueryForm()
form.set_choices()
if form.validate_on_submit():
query = Query(name=form.name.data,
sql=form.sql.data,
interval=form.interval.data,
platform=form.platform.data,
version=form.version.data,
description=form.description.data,
value=form.value.data,
removed=form.removed.data)
query.tags = create_tags(*form.tags.data.splitlines())
query.save()
return redirect(url_for('manage.query', query_id=query.id))
flash_errors(form)
return render_template('query.html', form=form)
def test_adding_and_then_removing_results_in_valid_configuration(self, node, testapp):
tag = Tag.create(value='foo')
node.tags.append(tag)
assert not node.get_config()['packs'] # should be an empty {}
assert not node.get_config()['schedule'] # should be an empty {}
query = Query.create(name='foobar', sql='select * from osquery_info;')
query.tags.append(tag)
query.save()
assert node.get_config()['schedule']
assert query.name in node.get_config()['schedule']
assert not node.get_config()['packs'] # should be an empty {}
def test_adding_and_then_removing_results_in_valid_configuration(
self, node, testapp):
tag = Tag.create(value='foo')
node.tags.append(tag)
assert not node.get_config()['packs'] # should be an empty {}
assert not node.get_config()['schedule'] # should be an empty {}
query = Query.create(name='foobar', sql='select * from osquery_info;')
query.tags.append(tag)
query.save()
assert node.get_config()['schedule']
assert query.name in node.get_config()['schedule']
assert not node.get_config()['packs'] # should be an empty {}
def create_query_pack_from_upload(upload):
'''
Create a pack and queries from a query pack file. **Note**, if a
pack already exists under the filename being uploaded, then any
queries defined here will be added to the existing pack! However,
if a query with a particular name already exists, and its sql is
NOT the same, then a new query with the same name but different id
will be created (as to avoid clobbering the existing query). If its
sql is identical, then the query will be reused.
'''
from doorman.models import Pack, Query
# The json package on Python 3 expects a `str` input, so we're going to
# read the body and possibly convert to the right type
body = upload.data.read()
if not isinstance(body, six.string_types):
body = body.decode('utf-8')
try:
data = json.loads(body)
except ValueError:
flash(u"Could not load pack as JSON - ensure it is JSON encoded",
'danger')
return None
else:
if 'queries' not in data:
flash(u"No queries in pack", 'danger')
return None
name = splitext(basename(upload.data.filename))[0]
pack = Pack.query.filter(Pack.name == name).first()
if not pack:
current_app.logger.debug("Creating pack %s", name)
pack = Pack.create(name=name, **data)
for query_name, query in data['queries'].items():
if not validate_osquery_query(query['query']):
flash('Invalid osquery query: "{0}"'.format(query['query']), 'danger')
return None
q = Query.query.filter(Query.name == query_name).first()
if not q:
q = Query.create(name=query_name, **query)
pack.queries.append(q)
current_app.logger.debug("Adding new query %s to pack %s",
q.name, pack.name)
continue
if q in pack.queries:
continue
if q.sql == query['query']:
current_app.logger.debug("Adding existing query %s to pack %s",
q.name, pack.name)
pack.queries.append(q)
else:
q2 = Query.create(name=query_name, **query)
current_app.logger.debug(
"Created another query named %s, but different sql: %r vs %r",
query_name, q2.sql.encode('utf-8'), q.sql.encode('utf-8'))
pack.queries.append(q2)
else:
pack.save()
flash(u"Imported query pack {0}".format(pack.name), 'success')
return pack