def __init__(self, application, request, **kwargs):
self.db = kwargs["db"]
del kwargs["db"]
super(SearchHandler, self).__init__(application, request, **kwargs)
self.log = logging.getLogger("galaxy.search_handler")
self.parser = Parser()
self.ip_fields = frozenset(["srcip", "dstip", "ip"])
def get_items(self):
from item.models import Item, user_items
if self.type == 'smart':
return Parser(Item, user_items).find({'query': self._query})
else:
return self.user.items.join(
Item.lists, aliased=True).filter(List.id == self.id)
class QueryShell(Cmd):
def __init__(self, table):
Cmd.__init__(self)
self.table = table
self.parser = Parser()
def do_select(self, cmd):
try:
t1 = time.time()
q = self.parser.parse(cmd)
t2 = time.time()
res = self.table.query(q)
t3 = time.time()
print "query parsed: ", q
print "Parse time (ms): ", ((t2 - t1)*1000)
print "Exec time (ms): ", ((t3 - t2)*1000)
print "%d results:", len(res)
print
print res
except Exception, e:
print "exception: ", e
traceback.print_exc(file=sys.stdout)
def parse(data):
query = {}
query['range'] = [0, 100]
if not 'group' in data:
query['sort'] = [{'key':'title', 'operator':'+'}]
for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
if key in data:
query[key] = data[key]
#print data
query['qs'] = Parser(models.Item, models.user_items).find(data)
if not 'group' in query:
query['qs'] = order(query['qs'], query['sort'])
return query
from utils import send_alert, speak, send_image, send_message
from camera import Camera
from processor import Processor
from queryparser import Parser
from celery import Celery
from config import PI_URL, MESSENGER_ID, LOG_PATH, ACCESS_TOKEN
app = Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
p = Processor()
parser = Parser()
@celery.task
def process_command(sender, message):
"""Background task to send an email with Flask-Mail."""
intent = parser.get_intent(message.lower())
args = parser.get_entities(message, intent)
print intent
print args
if intent == 'time':
send_message(sender, time.strftime('%I:%M %p'))
elif intent == 'expense' and 'interval' in args:
result = p.process(intent, args)
if args['exp_type'] == 'all':
def __init__(self, table):
Cmd.__init__(self)
self.table = table
self.parser = Parser()
class SearchHandler(BaseHandler):
def __init__(self, application, request, **kwargs):
self.db = kwargs["db"]
del kwargs["db"]
super(SearchHandler, self).__init__(application, request, **kwargs)
self.log = logging.getLogger("galaxy.search_handler")
self.parser = Parser()
self.ip_fields = frozenset(["srcip", "dstip", "ip"])
def initialize(self, *args, **kwargs):
super(SearchHandler, self).initialize(*args, **kwargs)
self.user = DEFAULT_USER
# Using the post() coroutine
def get(self, uri):
query_string = self.get_argument("q")
params = dict(start=self.get_argument("start", None),
end=self.get_argument("end", None))
es_query, parsed = self.parser.parse(query_string, params)
self.log.debug("es_query: %r" % es_query)
self.request.parsed = parsed
self.request.es_query = es_query
self.request.raw_query = query_string
self.request.body = json.dumps(es_query)
return self.post(uri)
def compare_searches(self, curr):
# See if the current search is similar enough to the previous to call them related
prev = self.db.execute("SELECT * FROM transcript " +\
"WHERE action='SEARCH' ORDER BY id DESC LIMIT 1").fetchone()
if not prev:
return None
# Split the raw_query into whitespaced tokens and compare
prev["data"] = json.loads(prev["data"])
prev_tokens = set(prev["data"]["raw_query"].split())
curr_tokens = set(curr["raw_query"].split())
self.log.debug("prev: %r, curr: %r" % (prev_tokens, curr_tokens))
if len(curr_tokens - prev_tokens) <= 2:
return prev["id"]
return None
def fixup(self, body):
body = json.loads(body)
self.log.debug("body: %r" % body)
self.log.debug("parsed: %r" % self.request.parsed)
if body.has_key("hits"):
for hit in body["hits"]["hits"]:
hit["_source"]["orig_@timestamp"] = hit["_source"][
"@timestamp"]
hit["_source"]["@timestamp"] = datetime.datetime.fromtimestamp(
int(hit["_source"]["@timestamp"]) / 1000).isoformat()
if body.has_key("aggregations"):
for rawfield, buckethash in body["aggregations"].iteritems():
fields = rawfield.split(",")
ipfields = []
for i, field in enumerate(fields):
if field in self.ip_fields:
ipfields.append(i)
self.log.debug("rawfield: %s, ipfields: %r" %
(rawfield, ipfields))
for bucket in buckethash["buckets"]:
if bucket.has_key("key_as_string"):
values = [bucket["key_as_string"]]
else:
values = str(bucket["key"]).split("\t")
newvalues = []
for i, value in enumerate(values):
if i in ipfields and "." not in value:
newvalues.append(
socket.inet_ntoa(struct.pack("!I",
int(value))))
else:
newvalues.append(value)
bucket["keys"] = newvalues
bucket["key"] = "-".join(newvalues)
# Build desc
desc = self.request.es_query["query"]["bool"]["must"][0]["query"][
"query_string"]["query"]
if self.request.parsed.has_key("groupby"):
desc += " (" + ",".join(self.request.parsed["groupby"][1:]) + ")"
desc = "[%d] " % body.get("hits", {}).get("total", 0) + desc
body = {
"results": body,
"query": self.request.parsed,
"raw_query": self.request.raw_query,
"es_query": self.request.es_query,
"description": desc
}
return body
def record(self, body):
data = {
"raw_query": self.request.raw_query,
"query": self.request.parsed,
"es_query": self.request.es_query
}
scope_id = self.get_argument("scope_id", None)
if scope_id:
scope_id = int(scope_id)
body["scope_id"] = scope_id
ref_id = self.get_argument("ref_id", None)
if ref_id:
body["ref_id"] = data["ref_id"] = int(ref_id)
elif scope_id:
body["ref_id"] = data["ref_id"] = int(scope_id)
else:
body["ref_id"] = data["ref_id"] = self.compare_searches(data)
self.log.debug("ref_id: %r" % body["ref_id"])
row = self.db.execute("SELECT data FROM transcript WHERE id=?",
(body["ref_id"], )).fetchone()
# if row:
# self.log.debug("row: %r" % row)
# body["referenced_search_description"] = json.loads(row["data"])["raw_query"]
if not scope_id:
scope_id = self.db.execute("SELECT id FROM scopes WHERE scope=?",
("default", )).fetchone()["id"]
# Log to results
body["results_id"] = self.db.execute("INSERT INTO results (user_id, results, timestamp) " +\
"VALUES ((SELECT id FROM users WHERE user=?),?,?)",
(DEFAULT_USER, base64.encodestring(zlib.compress(json.dumps(body))), time())).lastrowid
# id = self.db.execute("SELECT id FROM results " +\
# "WHERE user_id=(SELECT id FROM users WHERE user=?) " +\
# "ORDER BY id DESC LIMIT 1", (self.user,)).fetchone()
# body["results_id"] = id["id"]
body["transcript_id"] = self.db.execute("INSERT INTO transcript (user_id, action, data, description, " + \
"ref_id, scope_id, results_id, timestamp) " +\
"VALUES ((SELECT id FROM users WHERE user=?),?,?,?,?,?,?,?)",
(self.user, "SEARCH", json.dumps(data), body["description"],
body["ref_id"], scope_id, id["id"], time())).lastrowid
# newid = self.db.execute("SELECT id FROM transcript " +\
# "ORDER BY timestamp DESC LIMIT 1").fetchone()
# body["transcript_id"] = newid["id"]
return body
@tornado.web.gen.coroutine
def post(self, uri):
# Unless we explicitly want to intercept and federate, pass the req through
# to the first node listed in local_nodes conf
# query = self.request.get_argument("q", default=None)
# if not query:
# return self._bad_request("No q param given for query.")
self.request.host = self.passthrough_node
self.request.uri = "/es/_search"
uri = self.request.full_url()
req = HTTPRequest(uri,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
allow_nonstandard_methods=True)
self.log.debug("Passing req through %r" % req.url)
response = yield self.client.fetch(req, raise_error=False)
self.log.debug("got response: %r" % response)
if (response.error and
not isinstance(response.error, tornado.httpclient.HTTPError)):
self.set_status(500)
self.write('Internal server error:\n' + str(response.error))
else:
self.set_status(response.code, response.reason)
self._headers = tornado.httputil.HTTPHeaders(
) # clear tornado default header
for header, v in response.headers.get_all():
if header not in ('Content-Length', 'Transfer-Encoding',
'Content-Encoding', 'Connection'):
self.add_header(
header, v
) # some header appear multiple times, eg 'Set-Cookie'
if response.body:
# Apply any last minute field translations
fixedup_body = self.fixup(response.body)
fixedup_body = self.record(fixedup_body)
fixedup_body = json.dumps(fixedup_body)
self.set_header('Content-Length', len(fixedup_body))
self.write(fixedup_body)
self.finish()