def test_distributed_query_table_filter_status(self, db, testapp):
node1 = NodeFactory(host_identifier='node1')
node2 = NodeFactory(host_identifier='node2')
# Create a fake query and tasks for each node
q = DistributedQuery.create(
sql="select name, path, pid from processes where name = 'osqueryd';")
t1 = DistributedQueryTask.create(node=node1, distributed_query=q)
t2 = DistributedQueryTask.create(node=node2, distributed_query=q)
t1.update(status=DistributedQueryTask.PENDING)
t2.update(status=DistributedQueryTask.COMPLETE)
r1 = DistributedQueryResult.create(distributed_query_task=t1, distributed_query=q, columns={
'query': 'number 1',
})
r2 = DistributedQueryResult.create(distributed_query_task=t2, distributed_query=q, columns={
'query': 'number 2',
})
# Verify that only the complete one exists in the table
resp = testapp.get(url_for('manage.distributed_results', distributed_id=q.id, status='complete'))
assert 'number 1' not in resp.text
assert 'number 2' in resp.text
# Should only have one result
assert 'displaying <b>1 - 1</b> of <b>1</b> complete distributed query results' in resp.text
def distributed_write(node=None):
'''
'''
data = request.get_json()
current_app.logger.info("Got data: %s", data)
for guid, results in data.get('queries', {}).items():
task = DistributedQueryTask.query.filter(
DistributedQueryTask.guid == guid,
DistributedQueryTask.status == DistributedQueryTask.PENDING,
DistributedQueryTask.node == node,
).first()
if not task:
current_app.logger.error(
"Got result for distributed query not "
"in PENDING state: %s: %s", guid, json.dumps(data))
continue
for columns in results:
result = DistributedQueryResult(
columns,
distributed_query=task.distributed_query,
distributed_query_task=task)
db.session.add(result)
else:
task.status = DistributedQueryTask.COMPLETE
db.session.add(task)
else:
db.session.commit()
return jsonify(node_invalid=False)
def test_distributed_query_write_state_complete(self, db, node, testapp):
q = DistributedQuery.create(
sql="select name, path, pid from processes where name = 'osqueryd';")
t = DistributedQueryTask.create(node=node, distributed_query=q)
t.update(status=DistributedQueryTask.PENDING)
data = [{
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97830"
},
{
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97831"
}]
r = DistributedQueryResult.create(columns=data[0],
distributed_query=q,
distributed_query_task=t)
t.update(status=DistributedQueryTask.COMPLETE)
resp = testapp.post_json(url_for('api.distributed_write'), {
'node_key': node.node_key,
'queries': {
t.guid: '',
}
})
assert q.results
assert len(q.results) == 1
assert q.results[0] == r
assert q.results[0].columns == data[0]
def distributed_write(node=None):
'''
'''
data = request.get_json()
current_app.logger.info("Got data: %s", data)
for guid, data in data.get('queries', {}).items():
query = DistributedQuery.query.filter(
DistributedQuery.guid == guid,
DistributedQuery.status == DistributedQuery.PENDING,
DistributedQuery.node == node,
).first()
if not query:
current_app.logger.error(
"Got result for distributed query not "
"in PENDING state: %s: %s", guid, json.dumps(data))
continue
result = DistributedQueryResult(data, distributed_query=query)
query.status = DistributedQuery.COMPLETE
db.session.add(result, query)
else:
db.session.commit()
return jsonify(node_invalid=False)
def test_distributed_query_write_state_complete(self, db, node, testapp):
q = DistributedQuery.create(
sql=
"select name, path, pid from processes where name = 'osqueryd';",
node=node)
q.update(status=DistributedQuery.PENDING)
data = [{
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97830"
}, {
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97830"
}]
r = DistributedQueryResult.create(data=data, distributed_query=q)
q.update(status=DistributedQuery.COMPLETE)
resp = testapp.post_json(url_for('api.distributed_write'), {
'node_key': node.node_key,
'queries': {
q.guid: '',
}
})
assert q.result == r
assert q.result.data == data
def test_distributed_query_write_state_complete(self, db, node, testapp):
q = DistributedQuery.create(
sql="select name, path, pid from processes where name = 'osqueryd';",
node=node)
q.update(status=DistributedQuery.PENDING)
data = [{
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97830"
},
{
"name": "osqueryd",
"path": "/usr/local/bin/osqueryd",
"pid": "97830"
}]
r = DistributedQueryResult.create(data=data, distributed_query=q)
q.update(status=DistributedQuery.COMPLETE)
resp = testapp.post_json(url_for('api.distributed_write'), {
'node_key': node.node_key,
'queries': {
q.guid: '',
}
})
assert q.result == r
assert q.result.data == data
def distributed_write(node=None):
'''
'''
data = request.get_json()
if current_app.debug:
current_app.logger.debug(json.dumps(data, indent=2))
queries = data.get('queries', {})
statuses = data.get('statuses', {})
for guid, results in queries.items():
task = DistributedQueryTask.query.filter(
DistributedQueryTask.guid == guid,
DistributedQueryTask.status == DistributedQueryTask.PENDING,
DistributedQueryTask.node == node,
).first()
if not task:
current_app.logger.error(
"%s - Got result for distributed query not in PENDING "
"state: %s: %s", request.remote_addr, guid, json.dumps(data))
continue
# non-zero status indicates sqlite errors
if not statuses.get(guid, 0):
status = DistributedQueryTask.COMPLETE
else:
current_app.logger.error(
"%s - Got non-zero status code (%d) on distributed query %s",
request.remote_addr, statuses.get(guid), guid)
status = DistributedQueryTask.FAILED
for columns in results:
result = DistributedQueryResult(
columns,
distributed_query=task.distributed_query,
distributed_query_task=task)
db.session.add(result)
else:
task.status = status
db.session.add(task)
else:
# need to write last_checkin, last_ip on node
db.session.add(node)
db.session.commit()
return jsonify(node_invalid=False)
def test_distributed_query_table_basic(self, db, node, testapp):
# Create two fake queries, tasks, and fake results.
q1 = DistributedQuery.create(
sql="select name, path, pid from processes where name = 'osqueryd';")
t1 = DistributedQueryTask.create(node=node, distributed_query=q1)
q2 = DistributedQuery.create(
sql="select name, path, pid from processes where name = 'otherproc';")
t2 = DistributedQueryTask.create(node=node, distributed_query=q2)
t1.update(status=DistributedQueryTask.PENDING)
t2.update(status=DistributedQueryTask.PENDING)
r1 = DistributedQueryResult.create(distributed_query_task=t1, distributed_query=q1, columns={
'query': 'number 1',
})
r2 = DistributedQueryResult.create(distributed_query_task=t2, distributed_query=q2, columns={
'query': 'number 2',
})
# Verify that the first query is there, and the second is not
resp = testapp.get(url_for('manage.distributed_results', distributed_id=q1.id))
assert self.html_escape(q1.sql) in resp.text
assert self.html_escape(q2.sql) not in resp.text