Beispiel #1
0
 def __init__(self, application, request, **kwargs):
     self.db = kwargs["db"]
     del kwargs["db"]
     super(SearchHandler, self).__init__(application, request, **kwargs)
     self.log = logging.getLogger("galaxy.search_handler")
     self.parser = Parser()
     self.ip_fields = frozenset(["srcip", "dstip", "ip"])
Beispiel #2
0
 def get_items(self):
     from item.models import Item, user_items
     if self.type == 'smart':
         return Parser(Item, user_items).find({'query': self._query})
     else:
         return self.user.items.join(
             Item.lists, aliased=True).filter(List.id == self.id)
Beispiel #3
0
class QueryShell(Cmd):
    def __init__(self, table):
        Cmd.__init__(self)
        self.table = table
        self.parser = Parser()
    
    def do_select(self, cmd):
        try:
            t1 = time.time()
            q = self.parser.parse(cmd)
            t2 = time.time()
            res = self.table.query(q)
            t3 = time.time()

            print "query parsed: ", q
            print "Parse time (ms): ", ((t2 - t1)*1000)
            print "Exec  time (ms): ", ((t3 - t2)*1000)

            print "%d results:", len(res)
            print 
            print res
            
        except Exception, e:
            print "exception: ", e
            traceback.print_exc(file=sys.stdout)
Beispiel #4
0
def parse(data):
    query = {}
    query['range'] = [0, 100]
    if not 'group' in data:
        query['sort'] = [{'key':'title', 'operator':'+'}]
    for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
        if key in data:
            query[key] = data[key]
    #print data
    query['qs'] = Parser(models.Item, models.user_items).find(data)
    if not 'group' in query:
        query['qs'] = order(query['qs'], query['sort'])
    return query
Beispiel #5
0
from utils import send_alert, speak, send_image, send_message
from camera import Camera
from processor import Processor
from queryparser import Parser
from celery import Celery

from config import PI_URL, MESSENGER_ID, LOG_PATH, ACCESS_TOKEN

app = Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'

celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])

p = Processor()
parser = Parser()


@celery.task
def process_command(sender, message):
    """Background task to send an email with Flask-Mail."""

    intent = parser.get_intent(message.lower())
    args = parser.get_entities(message, intent)
    print intent
    print args
    if intent == 'time':
        send_message(sender, time.strftime('%I:%M %p'))
    elif intent == 'expense' and 'interval' in args:
        result = p.process(intent, args)
        if args['exp_type'] == 'all':
Beispiel #6
0
 def __init__(self, table):
     Cmd.__init__(self)
     self.table = table
     self.parser = Parser()
Beispiel #7
0
class SearchHandler(BaseHandler):
    def __init__(self, application, request, **kwargs):
        self.db = kwargs["db"]
        del kwargs["db"]
        super(SearchHandler, self).__init__(application, request, **kwargs)
        self.log = logging.getLogger("galaxy.search_handler")
        self.parser = Parser()
        self.ip_fields = frozenset(["srcip", "dstip", "ip"])

    def initialize(self, *args, **kwargs):
        super(SearchHandler, self).initialize(*args, **kwargs)
        self.user = DEFAULT_USER

    # Using the post() coroutine
    def get(self, uri):
        query_string = self.get_argument("q")
        params = dict(start=self.get_argument("start", None),
                      end=self.get_argument("end", None))
        es_query, parsed = self.parser.parse(query_string, params)
        self.log.debug("es_query: %r" % es_query)
        self.request.parsed = parsed
        self.request.es_query = es_query
        self.request.raw_query = query_string
        self.request.body = json.dumps(es_query)
        return self.post(uri)

    def compare_searches(self, curr):
        # See if the current search is similar enough to the previous to call them related
        prev = self.db.execute("SELECT * FROM transcript " +\
         "WHERE action='SEARCH' ORDER BY id DESC LIMIT 1").fetchone()
        if not prev:
            return None
        # Split the raw_query into whitespaced tokens and compare
        prev["data"] = json.loads(prev["data"])
        prev_tokens = set(prev["data"]["raw_query"].split())
        curr_tokens = set(curr["raw_query"].split())
        self.log.debug("prev: %r, curr: %r" % (prev_tokens, curr_tokens))
        if len(curr_tokens - prev_tokens) <= 2:
            return prev["id"]
        return None

    def fixup(self, body):
        body = json.loads(body)
        self.log.debug("body: %r" % body)
        self.log.debug("parsed: %r" % self.request.parsed)
        if body.has_key("hits"):
            for hit in body["hits"]["hits"]:
                hit["_source"]["orig_@timestamp"] = hit["_source"][
                    "@timestamp"]
                hit["_source"]["@timestamp"] = datetime.datetime.fromtimestamp(
                    int(hit["_source"]["@timestamp"]) / 1000).isoformat()
        if body.has_key("aggregations"):
            for rawfield, buckethash in body["aggregations"].iteritems():
                fields = rawfield.split(",")
                ipfields = []
                for i, field in enumerate(fields):
                    if field in self.ip_fields:
                        ipfields.append(i)
                self.log.debug("rawfield: %s, ipfields: %r" %
                               (rawfield, ipfields))

                for bucket in buckethash["buckets"]:
                    if bucket.has_key("key_as_string"):
                        values = [bucket["key_as_string"]]
                    else:
                        values = str(bucket["key"]).split("\t")
                    newvalues = []
                    for i, value in enumerate(values):
                        if i in ipfields and "." not in value:
                            newvalues.append(
                                socket.inet_ntoa(struct.pack("!I",
                                                             int(value))))
                        else:
                            newvalues.append(value)
                    bucket["keys"] = newvalues
                    bucket["key"] = "-".join(newvalues)

        # Build desc
        desc = self.request.es_query["query"]["bool"]["must"][0]["query"][
            "query_string"]["query"]
        if self.request.parsed.has_key("groupby"):
            desc += " (" + ",".join(self.request.parsed["groupby"][1:]) + ")"
        desc = "[%d] " % body.get("hits", {}).get("total", 0) + desc

        body = {
            "results": body,
            "query": self.request.parsed,
            "raw_query": self.request.raw_query,
            "es_query": self.request.es_query,
            "description": desc
        }

        return body

    def record(self, body):
        data = {
            "raw_query": self.request.raw_query,
            "query": self.request.parsed,
            "es_query": self.request.es_query
        }

        scope_id = self.get_argument("scope_id", None)
        if scope_id:
            scope_id = int(scope_id)
            body["scope_id"] = scope_id
        ref_id = self.get_argument("ref_id", None)
        if ref_id:
            body["ref_id"] = data["ref_id"] = int(ref_id)
        elif scope_id:
            body["ref_id"] = data["ref_id"] = int(scope_id)
        else:
            body["ref_id"] = data["ref_id"] = self.compare_searches(data)
            self.log.debug("ref_id: %r" % body["ref_id"])
            row = self.db.execute("SELECT data FROM transcript WHERE id=?",
                                  (body["ref_id"], )).fetchone()
            # if row:
            # 	self.log.debug("row: %r" % row)
            # 	body["referenced_search_description"] = json.loads(row["data"])["raw_query"]

        if not scope_id:
            scope_id = self.db.execute("SELECT id FROM scopes WHERE scope=?",
                                       ("default", )).fetchone()["id"]

        # Log to results
        body["results_id"] = self.db.execute("INSERT INTO results (user_id, results, timestamp) " +\
         "VALUES ((SELECT id FROM users WHERE user=?),?,?)",
         (DEFAULT_USER, base64.encodestring(zlib.compress(json.dumps(body))), time())).lastrowid
        # id = self.db.execute("SELECT id FROM results " +\
        # 	"WHERE user_id=(SELECT id FROM users WHERE user=?) " +\
        # 	"ORDER BY id DESC LIMIT 1", (self.user,)).fetchone()
        # body["results_id"] = id["id"]

        body["transcript_id"] = self.db.execute("INSERT INTO transcript (user_id, action, data, description, " + \
         "ref_id, scope_id, results_id, timestamp) " +\
         "VALUES ((SELECT id FROM users WHERE user=?),?,?,?,?,?,?,?)",
         (self.user, "SEARCH", json.dumps(data), body["description"],
          body["ref_id"], scope_id, id["id"], time())).lastrowid
        # newid = self.db.execute("SELECT id FROM transcript " +\
        # 	"ORDER BY timestamp DESC LIMIT 1").fetchone()
        # body["transcript_id"] = newid["id"]

        return body

    @tornado.web.gen.coroutine
    def post(self, uri):
        # Unless we explicitly want to intercept and federate, pass the req through
        #  to the first node listed in local_nodes conf

        # query = self.request.get_argument("q", default=None)
        # if not query:
        # 	return self._bad_request("No q param given for query.")

        self.request.host = self.passthrough_node
        self.request.uri = "/es/_search"
        uri = self.request.full_url()
        req = HTTPRequest(uri,
                          method=self.request.method,
                          body=self.request.body,
                          headers=self.request.headers,
                          follow_redirects=False,
                          allow_nonstandard_methods=True)

        self.log.debug("Passing req through %r" % req.url)
        response = yield self.client.fetch(req, raise_error=False)
        self.log.debug("got response: %r" % response)
        if (response.error and
                not isinstance(response.error, tornado.httpclient.HTTPError)):
            self.set_status(500)
            self.write('Internal server error:\n' + str(response.error))
        else:
            self.set_status(response.code, response.reason)
            self._headers = tornado.httputil.HTTPHeaders(
            )  # clear tornado default header

            for header, v in response.headers.get_all():
                if header not in ('Content-Length', 'Transfer-Encoding',
                                  'Content-Encoding', 'Connection'):
                    self.add_header(
                        header, v
                    )  # some header appear multiple times, eg 'Set-Cookie'

            if response.body:
                # Apply any last minute field translations
                fixedup_body = self.fixup(response.body)
                fixedup_body = self.record(fixedup_body)
                fixedup_body = json.dumps(fixedup_body)
                self.set_header('Content-Length', len(fixedup_body))
                self.write(fixedup_body)
        self.finish()