def convert2dasql(self, q): """ transform DASQuery object back into DASQL (human readable form) """ fields = q.mongo_query.get('fields', []) or [] exclude = das_record_keys() lookups = (f for f in fields if f not in exclude) das_conditions = q.mongo_query.get('spec', {}).items() short_daskey = lambda ldaskey: \ self.entity_names.get(ldaskey, ldaskey) dasql = ','.join(lookups) dasql += ' ' # space between lookup and conditions dasql += ' '.join('%s=%s' % (short_daskey(k), v) for k, v in das_conditions) # add post_filters if q.filters: dasql += ' | grep ' + ', '.join(q.filters) # not implemented: language specials - between, date # (however these are not supported by keyword search neither) # also aggregators (including unique) - left for user to implement return dasql.strip()
def get_from_cache(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves results from the cache" if dasquery.service_apis_map(): # valid DAS query result = self.get_das_records(dasquery, idx, limit, collection) for row in result: yield row else: # pure MongoDB query fields = dasquery.mongo_query.get('fields', []) if fields == None: fields = [] spec = dasquery.mongo_query.get('spec', {}) if dasquery.filters: if not fields: fields = [] fields += dasquery.filters pkeys = [k.split('.')[0] for k in fields] fields += das_record_keys() if 'records' in dasquery.query: fields = None # special case for DAS 'records' keyword skeys = self.mongo_sort_keys(collection, dasquery) result = self.get_records(collection, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) for row in result: if dasquery.filters: if pkeys and set(pkeys) & set(row.keys()): yield row else: yield row
def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves DAS records from the cache" msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection) self.logger.info(msg) idx = int(idx) fields, filter_cond = self.get_fields(dasquery) if fields == None: fields = [] if not fields: spec = dasquery.mongo_query.get('spec', {}) elif dasquery.hashes: spec = { 'qhash': { '$in': dasquery.hashes }, 'das.record': spec4data_records() } else: spec = {'qhash': dasquery.qhash, 'das.record': spec4data_records()} if filter_cond: spec.update(filter_cond) if 'records' in dasquery.query: fields = None # retrieve all fields for records DAS query else: # be sure to extract das internal keys fields += das_record_keys() # try to get sort keys all the time to get ordered list of # docs which allow unique_filter to apply afterwards skeys = self.mongo_sort_keys(collection, dasquery) res = self.get_records(collection, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) counter = 0 for row in res: counter += 1 yield row msg = 'qhash %s, found %s record(s) in %s collection' \ % (dasquery.qhash, counter, collection) print(dastimestamp('DAS INFO '), msg) if counter: msg = "yield %s record(s)" % counter self.logger.info(msg) # if no raw records were yield we look-up possible error records # and reset timestamp for record with system:['das'] if not counter: spec = {'qhash': dasquery.qhash} nrec = self.col.find(spec, **PYMONGO_OPTS).count() if nrec: msg = "for query %s, found %s non-result record(s)" \ % (dasquery, nrec) print(dastimestamp('DAS WARNING'), msg) for rec in self.col.find(spec, **PYMONGO_OPTS): if 'query' in rec: print(dastimestamp('DAS das record'), rec) self.update_das_expire(dasquery, etstamp())
def get_das_records(self, dasquery, idx=0, limit=0, collection='merge'): "Generator which retrieves DAS records from the cache" msg = "(%s, %s, %s, coll=%s)" % (dasquery, idx, limit, collection) self.logger.info(msg) idx = int(idx) fields, filter_cond = self.get_fields(dasquery) if fields == None: fields = [] if not fields: spec = dasquery.mongo_query.get('spec', {}) elif dasquery.hashes: spec = {'qhash':{'$in':dasquery.hashes}, 'das.record': spec4data_records()} else: spec = {'qhash':dasquery.qhash, 'das.record': spec4data_records()} if filter_cond: spec.update(filter_cond) if 'records' in dasquery.query: fields = None # retrieve all fields for records DAS query else: # be sure to extract das internal keys fields += das_record_keys() # try to get sort keys all the time to get ordered list of # docs which allow unique_filter to apply afterwards skeys = self.mongo_sort_keys(collection, dasquery) res = self.get_records(collection, spec, fields, skeys, \ idx, limit, dasquery.unique_filter) counter = 0 for row in res: counter += 1 yield row msg = 'qhash %s, found %s record(s) in %s collection' \ % (dasquery.qhash, counter, collection) print(dastimestamp('DAS INFO '), msg) if counter: msg = "yield %s record(s)" % counter self.logger.info(msg) # if no raw records were yield we look-up possible error records # and reset timestamp for record with system:['das'] if not counter: spec = {'qhash':dasquery.qhash} nrec = self.col.find(spec, **PYMONGO_OPTS).count() if nrec: msg = "for query %s, found %s non-result record(s)" \ % (dasquery, nrec) print(dastimestamp('DAS WARNING'), msg) for rec in self.col.find(spec, **PYMONGO_OPTS): if 'query' in rec: print(dastimestamp('DAS das record'), rec) self.update_das_expire(dasquery, etstamp())
def plainview(self, head, data): """ Represent data in DAS plain view for queries with filters. """ dasquery = head['dasquery'] fields = dasquery.mongo_query.get('fields', []) filters = [f for f in dasquery.filters if f != 'unique' and \ f.find('=') == -1 and f.find('<') == -1 and \ f.find('>') == -1] results = "" status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: results += 'ERROR: %s' % reason lookup_items = [i for i in fields if i not in das_record_keys()] for row in data: if filters: for flt in filters: try: for obj in DotDict(row).get_values(flt): results += str(obj) + ' ' except: pass results += '\n' else: for item in lookup_items: if item != lookup_items[0]: results += ', ' try: systems = row['das']['system'] mapkey = self.dasmapping.find_mapkey(systems[0], item) if not mapkey: mapkey = '%s.name' % item key, att = mapkey.split('.') if key in row: val = row[key] if isinstance(val, dict): results += val.get(att, '') elif isinstance(val, list): results += \ ' '.join(set([str(i.get(att, '')) for i in val])) except: pass results += '\n' # use DAS sort_rows function instead of python set, since we need to # preserve the order of records in final output rows = [r for r in results.split('\n') if r] results = '\n'.join([r for r in sort_rows(rows)]) return results