Example #1
0
    def convert2dasql(self, q):
        """
        transform DASQuery object back into DASQL (human readable form)
        """
        fields = q.mongo_query.get('fields', []) or []
        exclude = das_record_keys()
        lookups = (f for f in fields
                   if f not in exclude)

        das_conditions = q.mongo_query.get('spec', {}).items()

        short_daskey = lambda ldaskey: \
            self.entity_names.get(ldaskey, ldaskey)

        dasql = ','.join(lookups)
        dasql += ' '  # space between lookup and conditions
        dasql += ' '.join('%s=%s' % (short_daskey(k), v)
                          for k, v in das_conditions)

        # add post_filters
        if q.filters:
            dasql += ' | grep ' + ', '.join(q.filters)

        # not implemented:  language specials - between, date
        #  (however these are not supported by keyword search neither)
        # also aggregators (including unique) - left for user to implement
        return dasql.strip()
Example #2
0
 def get_from_cache(self, dasquery, idx=0, limit=0, collection='merge'):
     "Generator which retrieves results from the cache"
     if  dasquery.service_apis_map(): # valid DAS query
         result = self.get_das_records(dasquery, idx, limit, collection)
         for row in result:
             yield row
     else: # pure MongoDB query
         fields  = dasquery.mongo_query.get('fields', [])
         if  fields == None:
             fields = []
         spec    = dasquery.mongo_query.get('spec', {})
         if  dasquery.filters:
             if  not fields:
                 fields = []
             fields += dasquery.filters
             pkeys   = [k.split('.')[0] for k in fields]
         fields += das_record_keys()
         if  'records' in dasquery.query:
             fields = None # special case for DAS 'records' keyword
         skeys   = self.mongo_sort_keys(collection, dasquery)
         result  = self.get_records(collection, spec, fields, skeys, \
                         idx, limit, dasquery.unique_filter)
         for row in result:
             if  dasquery.filters:
                 if  pkeys and set(pkeys) & set(row.keys()):
                     yield row
             else:
                 yield row
Example #3
0
 def get_from_cache(self, dasquery, idx=0, limit=0, collection='merge'):
     "Generator which retrieves results from the cache"
     if dasquery.service_apis_map():  # valid DAS query
         result = self.get_das_records(dasquery, idx, limit, collection)
         for row in result:
             yield row
     else:  # pure MongoDB query
         fields = dasquery.mongo_query.get('fields', [])
         if fields == None:
             fields = []
         spec = dasquery.mongo_query.get('spec', {})
         if dasquery.filters:
             if not fields:
                 fields = []
             fields += dasquery.filters
             pkeys = [k.split('.')[0] for k in fields]
         fields += das_record_keys()
         if 'records' in dasquery.query:
             fields = None  # special case for DAS 'records' keyword
         skeys = self.mongo_sort_keys(collection, dasquery)
         result  = self.get_records(collection, spec, fields, skeys, \
                         idx, limit, dasquery.unique_filter)
         for row in result:
             if dasquery.filters:
                 if pkeys and set(pkeys) & set(row.keys()):
                     yield row
             else:
                 yield row
Example #4
0
    def convert2dasql(self, q):
        """
        transform DASQuery object back into DASQL (human readable form)
        """
        fields = q.mongo_query.get('fields', []) or []
        exclude = das_record_keys()
        lookups = (f for f in fields if f not in exclude)

        das_conditions = q.mongo_query.get('spec', {}).items()

        short_daskey = lambda ldaskey: \
            self.entity_names.get(ldaskey, ldaskey)

        dasql = ','.join(lookups)
        dasql += ' '  # space between lookup and conditions
        dasql += ' '.join('%s=%s' % (short_daskey(k), v)
                          for k, v in das_conditions)

        # add post_filters
        if q.filters:
            dasql += ' | grep ' + ', '.join(q.filters)

        # not implemented:  language specials - between, date
        #  (however these are not supported by keyword search neither)
        # also aggregators (including unique) - left for user to implement
        return dasql.strip()
Example #5
0
    def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'):
        "Generator which retrieves DAS records from the cache"
        msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection)
        self.logger.info(msg)

        idx = int(idx)
        fields, filter_cond = self.get_fields(dasquery)
        if fields == None:
            fields = []
        if not fields:
            spec = dasquery.mongo_query.get('spec', {})
        elif dasquery.hashes:
            spec = {
                'qhash': {
                    '$in': dasquery.hashes
                },
                'das.record': spec4data_records()
            }
        else:
            spec = {'qhash': dasquery.qhash, 'das.record': spec4data_records()}
        if filter_cond:
            spec.update(filter_cond)
        if 'records' in dasquery.query:
            fields = None  # retrieve all fields for records DAS query
        else:
            # be sure to extract das internal keys
            fields += das_record_keys()
        # try to get sort keys all the time to get ordered list of
        # docs which allow unique_filter to apply afterwards
        skeys = self.mongo_sort_keys(collection, dasquery)
        res     = self.get_records(collection, spec, fields, skeys, \
                        idx, limit, dasquery.unique_filter)
        counter = 0
        for row in res:
            counter += 1
            yield row
        msg = 'qhash %s, found %s record(s) in %s collection' \
                % (dasquery.qhash, counter, collection)
        print(dastimestamp('DAS INFO '), msg)

        if counter:
            msg = "yield %s record(s)" % counter
            self.logger.info(msg)

        # if no raw records were yield we look-up possible error records
        # and reset timestamp for record with system:['das']
        if not counter:
            spec = {'qhash': dasquery.qhash}
            nrec = self.col.find(spec, **PYMONGO_OPTS).count()
            if nrec:
                msg = "for query %s, found %s non-result record(s)" \
                        % (dasquery, nrec)
                print(dastimestamp('DAS WARNING'), msg)
                for rec in self.col.find(spec, **PYMONGO_OPTS):
                    if 'query' in rec:
                        print(dastimestamp('DAS das record'), rec)
            self.update_das_expire(dasquery, etstamp())
Example #6
0
    def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'):
        "Generator which retrieves DAS records from the cache"
        msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection)
        self.logger.info(msg)

        idx = int(idx)
        fields, filter_cond = self.get_fields(dasquery)
        if  fields == None:
            fields = []
        if  not fields:
            spec = dasquery.mongo_query.get('spec', {})
        elif dasquery.hashes:
            spec = {'qhash':{'$in':dasquery.hashes},
                    'das.record': spec4data_records()}
        else:
            spec = {'qhash':dasquery.qhash,
                    'das.record': spec4data_records()}
        if  filter_cond:
            spec.update(filter_cond)
        if  'records' in dasquery.query:
            fields  = None # retrieve all fields for records DAS query
        else:
            # be sure to extract das internal keys
            fields += das_record_keys()
        # try to get sort keys all the time to get ordered list of
        # docs which allow unique_filter to apply afterwards
        skeys   = self.mongo_sort_keys(collection, dasquery)
        res     = self.get_records(collection, spec, fields, skeys, \
                        idx, limit, dasquery.unique_filter)
        counter = 0
        for row in res:
            counter += 1
            yield row
        msg = 'qhash %s, found %s record(s) in %s collection' \
                % (dasquery.qhash, counter, collection)
        print(dastimestamp('DAS INFO '), msg)

        if  counter:
            msg = "yield %s record(s)" % counter
            self.logger.info(msg)

        # if no raw records were yield we look-up possible error records
        # and reset timestamp for record with system:['das']
        if  not counter:
            spec = {'qhash':dasquery.qhash}
            nrec = self.col.find(spec, **PYMONGO_OPTS).count()
            if  nrec:
                msg = "for query %s, found %s non-result record(s)" \
                        % (dasquery, nrec)
                print(dastimestamp('DAS WARNING'), msg)
                for rec in self.col.find(spec, **PYMONGO_OPTS):
                    if  'query' in rec:
                        print(dastimestamp('DAS das record'), rec)
            self.update_das_expire(dasquery, etstamp())
Example #7
0
 def plainview(self, head, data):
     """
     Represent data in DAS plain view for queries with filters.
     """
     dasquery = head['dasquery']
     fields   = dasquery.mongo_query.get('fields', [])
     filters  = [f for f in dasquery.filters if f != 'unique' and \
                     f.find('=') == -1 and f.find('<') == -1 and \
                     f.find('>') == -1]
     results  = ""
     status   = head.get('status', None)
     if  status == 'fail':
         reason = head.get('reason', '')
         if  reason:
             results += 'ERROR: %s' % reason
     lookup_items = [i for i in fields if i not in das_record_keys()]
     for row in data:
         if  filters:
             for flt in filters:
                 try:
                     for obj in DotDict(row).get_values(flt):
                         results += str(obj) + ' '
                 except:
                     pass
             results += '\n'
         else:
             for item in lookup_items:
                 if  item != lookup_items[0]:
                     results += ', '
                 try:
                     systems = row['das']['system']
                     mapkey  = self.dasmapping.find_mapkey(systems[0], item)
                     if  not mapkey:
                         mapkey = '%s.name' % item
                     key, att = mapkey.split('.')
                     if  key in row:
                         val = row[key]
                         if  isinstance(val, dict):
                             results += val.get(att, '')
                         elif isinstance(val, list):
                             results += \
                             ' '.join(set([str(i.get(att, '')) for i in val]))
                 except:
                     pass
             results += '\n'
     # use DAS sort_rows function instead of python set, since we need to
     # preserve the order of records in final output
     rows = [r for r in results.split('\n') if r]
     results = '\n'.join([r for r in sort_rows(rows)])
     return results
Example #8
0
 def plainview(self, head, data):
     """
     Represent data in DAS plain view for queries with filters.
     """
     dasquery = head['dasquery']
     fields   = dasquery.mongo_query.get('fields', [])
     filters  = [f for f in dasquery.filters if f != 'unique' and \
                     f.find('=') == -1 and f.find('<') == -1 and \
                     f.find('>') == -1]
     results  = ""
     status   = head.get('status', None)
     if  status == 'fail':
         reason = head.get('reason', '')
         if  reason:
             results += 'ERROR: %s' % reason
     lookup_items = [i for i in fields if i not in das_record_keys()]
     for row in data:
         if  filters:
             for flt in filters:
                 try:
                     for obj in DotDict(row).get_values(flt):
                         results += str(obj) + ' '
                 except:
                     pass
             results += '\n'
         else:
             for item in lookup_items:
                 if  item != lookup_items[0]:
                     results += ', '
                 try:
                     systems = row['das']['system']
                     mapkey  = self.dasmapping.find_mapkey(systems[0], item)
                     if  not mapkey:
                         mapkey = '%s.name' % item
                     key, att = mapkey.split('.')
                     if  key in row:
                         val = row[key]
                         if  isinstance(val, dict):
                             results += val.get(att, '')
                         elif isinstance(val, list):
                             results += \
                             ' '.join(set([str(i.get(att, '')) for i in val]))
                 except:
                     pass
             results += '\n'
     # use DAS sort_rows function instead of python set, since we need to
     # preserve the order of records in final output
     rows = [r for r in results.split('\n') if r]
     results = '\n'.join([r for r in sort_rows(rows)])
     return results