def run_server(self):
from gevent import pywsgi
from werkzeug.debug import DebuggedApplication
from geventwebsocket.handler import WebSocketHandler
validate_osquery_query('select * from processes;')
pywsgi.WSGIServer(('', 5000), DebuggedApplication(app),
handler_class=WebSocketHandler, keyfile='../nginx/private.key',
certfile='../nginx/certificate.crt').serve_forever()
def post(self):
args = self.parser.parse_args(
) # need to exists for input payload validation
name = args['name']
sql = args['query']
interval = args['interval']
tags = args['tags'].split(',')
query = dao.get_query_by_name(name)
if query:
message = 'Query with this name already exists'
elif not validate_osquery_query(sql):
message = ('Invalid osquery query: "{0}"'.format(args['query']))
else:
query = dao.create_query_obj(name,
sql,
interval,
args['platform'],
args['version'],
args['description'],
args['value'],
100,
snapshot=args['snapshot'])
if tags:
query.tags = create_tags(*tags)
query.save()
return marshal({'query_id': query.id}, wrapper.add_query_wrapper)
return marshal(respcls(message), parentwrapper.failure_response_parent)
def post(self, query_id):
args = self.parser.parse_args()
if args['snapshot'] == "true":
args['snapshot'] = True
else:
args['snapshot'] = False
if args['tags']:
args['tags'] = args['tags'].split(',')
else:
args['tags'] = []
if query_id:
query = dao.get_query_by_id(query_id)
if query:
if validate_osquery_query(args['query']):
query = dao.edit_query_by_id(query, args)
return marshal(
respcls(
"Successfully updated the query for the given id",
"success", marshal(query, wrapper.query_wrapper)),
parentwrapper.common_response_wrapper)
else:
message = "Query is not a valid SQL!"
else:
message = "Query with this id does not exist"
else:
message = "Missing query id"
return marshal(respcls(message), parentwrapper.failure_response_parent)
def test_custom_schema(self):
query = 'SELECT * FROM custom_table;'
assert validate_osquery_query(query) is False
try:
# This is a bit hacky, but it clears the mock DB so the next call
# will re-create it.
osquery_mock_db.db = None
current_app.config['POLYLOGYX_EXTRA_SCHEMA'].append(
'CREATE TABLE custom_table (id INTEGER);'
)
assert validate_osquery_query(query) is True
finally:
# Remove final entry from the config.
current_app.config['POLYLOGYX_EXTRA_SCHEMA'] = \
current_app.config['POLYLOGYX_EXTRA_SCHEMA'][:-1]
def add_pack_through_json_data(args):
from polylogyx.wrappers.v1 import parent_wrappers
from polylogyx.utils import create_tags, validate_osquery_query
from flask_restplus import marshal
if 'tags' in args:
tags = args['tags'].split(',')
else:
tags = []
name = args['name']
queries = args['queries']
category = args['category']
platform = args.get('platform', None)
version = args.get('version', None)
description = args.get('description', None)
shard = args.get('shard', None)
pack = packs_dao.get_pack_by_name(name)
if not pack:
pack = packs_dao.add_pack(name, category, platform, version,
description, shard)
for query_name, query in queries.items():
if not validate_osquery_query(query['query']):
message = ('Invalid osquery query: "{0}"'.format(query['query']))
return marshal({'message': message},
parent_wrappers.failure_response_parent)
q = queries_dao.get_query_by_name(query_name)
if not q:
q = queries_dao.add_query(query_name, **query)
pack.queries.append(q)
current_app.logger.debug("Adding new query %s to pack %s", q.name,
pack.name)
continue
else:
if q.sql == query['query']:
current_app.logger.debug("Adding existing query %s to pack %s",
q.name, pack.name)
pack.queries.append(q)
else:
q2 = queries_dao.add_query(query_name, **query)
current_app.logger.debug(
"Created another query named %s, but different sql: %r vs %r",
query_name, q2.sql.encode('utf-8'), q.sql.encode('utf-8'))
pack.queries.append(q2)
if q in pack.queries:
continue
if pack:
if tags:
pack.tags = create_tags(*tags)
pack.save()
return pack
def post(self):
from polylogyx.dao.v1 import packs_dao
args = self.parser.parse_args()
name = args['name']
sql = args['query']
interval = args['interval']
if args['snapshot'] == "true":
args['snapshot'] = True
else:
args['snapshot'] = False
if args['tags']:
tags = args['tags'].split(',')
else:
tags = args['tags']
packs = []
if args['packs']: packs = args['packs'].split(',')
query = dao.get_query_by_name(name)
if query:
message = 'Query with this name already exists'
elif not validate_osquery_query(sql):
message = ('Invalid osquery query: "{0}"'.format(args['query']))
elif not is_number_positive(interval):
message = 'Interval provided is not valid! Please provide an inverval greater than 0'
else:
query = dao.create_query_obj(name,
sql,
interval,
args['platform'],
args['version'],
args['description'],
args['value'],
100,
snapshot=args['snapshot'])
if tags:
query.tags = create_tags(*tags)
if packs:
packs_list = []
for pack_name in packs:
pack = packs_dao.get_pack_by_name(pack_name)
if pack:
packs_list.append(pack)
query.packs = packs_list
query.save()
return marshal({'query_id': query.id}, wrapper.add_query_wrapper)
return marshal(respcls(message), parentwrapper.failure_response_parent)
def add_pack_through_json_data(args):
from polylogyx.dao import packs_dao, queries_dao
from polylogyx.wrappers import parent_wrappers
from polylogyx.utils import create_tags, validate_osquery_query
from flask_restplus import marshal
tags = args['tags']
name = args['name']
queries = args['queries']
pack = packs_dao.get_pack_by_name(name)
if not pack:
pack = packs_dao.add_pack(**args)
for query_name, query in queries.items():
if not validate_osquery_query(query['query']):
message = ('Invalid osquery query: "{0}"'.format(query['query']))
return marshal({'message': message}, parent_wrappers.failure_response_parent)
q = queries_dao.get_query_by_name(query_name)
if not q:
q = queries_dao.add_query(query_name, **query)
pack.queries.append(q)
current_app.logger.debug("Adding new query %s to pack %s",
q.name, pack.name)
continue
if q in pack.queries:
continue
if q.sql == query['query']:
current_app.logger.debug("Adding existing query %s to pack %s",
q.name, pack.name)
pack.queries.append(q)
else:
q2 = queries_dao.add_query(query_name, **query)
current_app.logger.debug(
"Created another query named %s, but different sql: %r vs %r",
query_name, q2.sql.encode('utf-8'), q.sql.encode('utf-8'))
pack.queries.append(q2)
if pack:
if tags:
pack.tags = create_tags(*tags)
pack.save()
return pack
def __call__(self, form, field):
if not validate_osquery_query(field.data):
raise ValidationError(self.message)
import socketio
from polylogyx.utils import validate_osquery_query
constant_file_event = 'win_file_events'
constant_process_event = 'win_process_events'
app = create_app(config=CurrentConfig)
async_mode = 'gevent'
sio = socketio.Server(logger=True, async_mode=async_mode)
sockets = Sockets(app)
app.wsgi_app = socketio.Middleware(sio, app.wsgi_app)
with app.app_context():
validate_osquery_query('select * from processes;')
def _make_context():
return {'app': app, 'db': db}
class SSLServer(Command):
def run_server(self):
from gevent import pywsgi
from werkzeug.debug import DebuggedApplication
from geventwebsocket.handler import WebSocketHandler
validate_osquery_query('select * from processes;')
pywsgi.WSGIServer(('', 5000),
DebuggedApplication(app),
def post(self, node=None):
args = self.parser.parse_args(
) # need to exists for input payload validation
onlineNodes = 0
sql = args['query']
if not validate_osquery_query(sql):
message = u'Field must contain valid SQL to be run against osquery tables'
else:
status = 'success'
message = 'Successfully send the distributed query'
tag_all = 'all'
current_app.logger.info(
"%s - %s checking in for distributed query",
request.remote_addr, node)
# all nodes get this query
nodes = []
tags = []
if args['tags']:
tags = args['tags'].split(',')
if args['nodes']:
nodeKeyList = args['nodes'].split(',')
else:
nodeKeyList = []
if not nodeKeyList and not tags:
# all nodes get this query
nodes = nodedao.get_all_nodes()
if nodeKeyList:
nodes = nodedao.extendNodesByNodeKeyList(nodeKeyList)
if tags:
nodes = nodedao.extendNodesByTag(tags)
query = dao.add_distributed_query(sql, args['description'])
win_sql_query = None
typed_query = query.sql
query_windows_specific = False
if 'win_file_events' in query.sql or 'win_process_events' in query.sql:
win_sql_query = typed_query
query_windows_specific = True
elif 'file_events' in query.sql:
win_sql_query = query.sql.replace('file_events',
'win_file_events')
elif 'process_events' in query.sql:
win_sql_query = query.sql.replace('process_events',
'win_process_events')
for node in nodes:
if node_is_active(node):
onlineNodes += 1
task = dao.create_distributed_task_obj(node, query)
if node.platform == 'windows':
task.sql = win_sql_query
if not (node.platform != 'windows'
and query_windows_specific):
db.session.add(task)
else:
db.session.commit()
if onlineNodes == 0:
message = 'No active node present'
else:
return marshal({'query_id': query.id},
wrapper.add_query_wrapper)
return marshal(respcls(message), parentwrapper.failure_response_parent)
def post(self, node=None):
from manage import declare_queue
args = self.parser.parse_args(
) # need to exists for input payload validation
onlineNodes = 0
hosts_array = []
sql = args['query']
if not validate_osquery_query(sql):
message = u'Field must contain valid SQL to be run against osquery tables'
else:
status = 'success'
message = 'Distributed query is sent successfully'
tag_all = 'all'
current_app.logger.info(
"%s - %s checking in for distributed query",
request.remote_addr, node)
# all nodes get this query
nodes = []
tags = []
if args['tags']:
tags = args['tags'].split(',')
if args['nodes']:
nodeKeyList = args['nodes'].split(',')
else:
nodeKeyList = []
if not nodeKeyList and not tags:
# all nodes get this query
nodes = nodedao.get_all_nodes()
if nodeKeyList:
nodes.extend(nodedao.extendNodesByNodeKeyList(nodeKeyList))
if tags:
nodes.extend(nodedao.extendNodesByTag(tags))
query = dao.add_distributed_query(sql, args['description'])
if nodes:
for node in nodes:
if node.node_is_active():
onlineNodes += 1
hosts_array.append({
"host_identifier": node.host_identifier,
"hostname": node.display_name,
"node_id": node.id
})
task = dao.create_distributed_task_obj(node, query)
db.session.add(task)
else:
db.session.commit()
declare_queue(query.id)
if onlineNodes == 0:
message = 'No active node present'
else:
current_app.logger.info(
"Distributed Query {0} is added to the hosts {1}".format(
query.id,
[host['host_identifier'] for host in hosts_array]))
return marshal(
respcls(
message, status, {
'query_id': query.id,
'onlineNodes': onlineNodes,
'online_nodes_details': hosts_array
}), parentwrapper.common_response_wrapper)
return marshal(respcls(message), parentwrapper.failure_response_parent)
def test_bad_table(self):
query = 'SELECT * FROM a_table_that_does_not_exist;'
assert validate_osquery_query(query) is False
def test_syntax_error(self):
query = 'SELECT * FROM'
assert validate_osquery_query(query) is False
def test_complex_validate(self):
# From Facebook's blog post: https://code.facebook.com/posts/844436395567983/introducing-osquery/
query = 'SELECT DISTINCT process.name, listening.port, listening.address, process.pid FROM processes AS process JOIN listening_ports AS listening ON process.pid = listening.pid;'
assert validate_osquery_query(query) is True
def test_simple_validate(self):
query = 'SELECT * FROM osquery_info;'
assert validate_osquery_query(query) is True
