def on_start(self):
        super(DiscoveryService, self).on_start()

        cfg_datastore = CFG.get_safe('container.datastore.default_server')
        if cfg_datastore != "postgresql":
            raise Exception("Discovery service does not support datastores other than postgresql")

        self.ds_discovery = DatastoreDiscovery(self)
    def on_start(self): # pragma no cover
        super(DiscoveryService,self).on_start()

        cfg_datastore = CFG.get_safe('container.datastore.default_server')
        if cfg_datastore != "postgresql":
            raise Exception("Discovery service does not support datastores other than postgresql")

        self.ds_discovery = DatastoreDiscovery(self)
    def on_start(self): # pragma no cover
        super(DiscoveryService,self).on_start()

        self.use_es = CFG.get_safe('system.elasticsearch', False)

        self.elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
        self.elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')

        self.ep = EventPublisher(event_type = 'SearchBufferExceededEvent')
        self.heuristic_cutoff = 4

        self.cfg_datastore = CFG.get_safe('container.datastore.default_server', "couchdb")
        self.ds_discovery = None
        if self.cfg_datastore != "couchdb":
            self.ds_discovery = DatastoreDiscovery(self)
class DiscoveryService(BaseDiscoveryService):
    MAX_SEARCH_RESULTS=CFG.get_safe('service.discovery.max_search_results', 250)

    def on_start(self): # pragma no cover
        super(DiscoveryService,self).on_start()

        cfg_datastore = CFG.get_safe('container.datastore.default_server')
        if cfg_datastore != "postgresql":
            raise Exception("Discovery service does not support datastores other than postgresql")

        self.ds_discovery = DatastoreDiscovery(self)

   
    #===================================================================
    # Query Methods
    #===================================================================

    def query(self, query=None, id_only=True):
        """Issue a query against the indexes as specified in the query, applying filters and operators
        accordingly. The query format is a structured dict.
        See the query format definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format

        @param query    dict
        @param id_only    bool
        @retval results    list
        """
        validate_true(query, 'Invalid query')

        return self.request(query, id_only)

    def parse(self, search_request='', id_only=True):
        """Parses a given string request and assembles the query, processes the query and returns the results of the query.
        This is the primary means of interfacing with the search features in discovery.
        See the query language definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format

        @param search_request    str
        @param id_only    bool
        @retval results    list
        """
        log.info("Search DSL: %s", search_request)
        query_request = self._parse_query_string(search_request)
        return self.request(query_request, id_only=id_only)

    def _parse_query_string(self, query_string):
        """Given a query string in Discovery service DSL, parse and return query structure"""
        parser = QueryLanguage()
        query_request = parser.parse(query_string)
        return query_request

    def request(self, query=None, id_only=True):
        if not query:
            raise BadRequest('No request query provided')

        if "QUERYEXP" in query and self.ds_discovery:
            # Support for datastore queries
            pass

        elif 'query' not in query:
            raise BadRequest('Unsuported request. %s' % query)

        res = self.ds_discovery.execute_query(query, id_only=id_only)
        return res


    #===================================================================
    # Special Query Methods
    #===================================================================

    def query_association(self, resource_id='', depth=0, id_only=False):
        validate_true(resource_id, 'Unspecified resource')
        if depth:
            resource_ids = self.iterative_traverse(resource_id, depth-1)
        else:
            resource_ids = self.traverse(resource_id)
        if id_only:
            return resource_ids

        if not isinstance(resource_ids, list):
            resource_ids = list(resource_ids)
        resources = self.clients.resource_registry.read_mult(resource_ids)

        return resources

    def query_owner(self, resource_id='', depth=0, id_only=False):
        validate_true(resource_id, 'Unspecified resource')
        if depth:
            resource_ids = self.iterative_traverse(resource_id, depth-1)
        else:
            resource_ids = self.reverse_traverse(resource_id)
        if id_only:
            return resource_ids

        if not isinstance(resource_ids, list):
            resource_ids = list(resource_ids)
        resources = self.clients.resource_registry.read_mult(resource_ids)

        return resources

    def query_collection(self,collection_id='', id_only=False):
        validate_true(collection_id, 'Unspecified collection id')
        resource_ids = self.clients.index_management.list_collection_resources(collection_id, id_only=True)
        if id_only:
            return resource_ids
        
        resources = map(self.clients.resource_registry.read,resource_ids)
        return resources


    def traverse(self, resource_id=''):
        """Breadth-first traversal of the association graph for a specified resource.

        @param resource_id    str
        @retval resources    list
        """

        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_objects_mult(subjects=resource_ids,id_only=True)[0]

        visited_resources = deque(edges([resource_id]))
        traversal_queue = deque()
        done = False
        t = None
        while not done:
            t = traversal_queue or deque(visited_resources)
            traversal_queue = deque()
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    traversal_queue.append(e)
            if not len(traversal_queue): done = True

        return list(visited_resources)

    def reverse_traverse(self, resource_id=''):
        """Breadth-first traversal of the association graph for a specified resource.

        @param resource_id    str
        @retval resources    list
        """

        def edges(resource_ids=[]):
            if not isinstance(resource_ids,list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_subjects_mult(objects=resource_ids,id_only=True)[0]

        visited_resources = deque(edges([resource_id]))
        traversal_queue = deque()
        done = False
        t = None
        while not done:
            t = traversal_queue or deque(visited_resources)
            traversal_queue = deque()
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    traversal_queue.append(e)
            if not len(traversal_queue): done = True



        return list(visited_resources)


    def iterative_traverse(self, resource_id='', limit=-1):
        '''
        Iterative breadth first traversal of the resource associations
        '''
        #--------------------------------------------------------------------------------
        # Retrieve edges for this resource
        #--------------------------------------------------------------------------------
        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_objects_mult(subjects=resource_ids,id_only=True)[0]

        gathered = deque()
        visited_resources = deque(edges([resource_id]))
        while limit>0:
            t = gathered or deque(visited_resources)
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    gathered.append(e)
            if not len(gathered): break

            t = deque(gathered)
            gathered = deque()
            limit -= 1

        return list(visited_resources)

    def iterative_reverse_traverse(self, resource_id='', limit=-1):
        '''
        Iterative breadth first traversal of the resource associations
        '''
        #--------------------------------------------------------------------------------
        # Retrieve edges for this resource
        #--------------------------------------------------------------------------------
        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_subjects_mult(objects=resource_ids,id_only=True)[0]

        gathered = deque()
        visited_resources = deque(edges([resource_id]))
        while limit>0:
            t = gathered or deque(visited_resources)
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    gathered.append(e)
            if not len(gathered): break

            t = deque(gathered)
            gathered = deque()
            limit -= 1

        return list(visited_resources)


    #===================================================================
    # View Management
    #===================================================================

    def create_view(self, view_name='', description='', fields=None, order=None, filters=''):
        """Creates a view which has the specified search fields, the order in which the search fields are presented
        to a query and a term filter.
        @param view_name Name of the view
        @param description Simple descriptive sentence
        @param fields Search fields
        @param order List of fields to determine order of precendence in which the results are presented
        @param filters Simple term filter
        """
        res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
        if len(res) > 0:
            raise BadRequest('The view resource with name: %s, already exists.' % view_name)

        #======================
        # Arg Validations
        #======================
        validate_is_instance(fields,list, 'Specified fields must be a list.')
        validate_true(len(fields)>0, 'Specfied fields must be a list.')
        if order is not None:
            validate_is_instance(order,list, 'Specified order must be a list of fields')
            for field in order:
                if not field in fields:
                    raise BadRequest('The specified ordering field was not part of the search fields.')

        fields = set(fields) # Convert fields to a set for aggregation across the catalogs
        #======================================================================================================
        # Priorty Queue Index Matching
        #======================================================================================================

        pq = [] # Priority queue for matching
        catalog_id = None
        catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
        for catalog in catalogs:
            if set(catalog.catalog_fields).issubset(fields):
                index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
                heapq.heappush(pq, (index_num,catalog))
        if pq:
            weight, catalog = heapq.heappop(pq)
            if weight < self.heuristic_cutoff:
                catalog_id = catalog._id


        if catalog_id is None:
            catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))

        view_res = View(name=view_name, description=description)
        view_res.order = order
        view_res.filters = filters
        view_id, _ = self.clients.resource_registry.create(view_res)
        self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
        return view_id

    def read_view(self, view_id=''):
        return self.clients.resource_registry.read(view_id)

    def update_view(self, view=None):
        self.clients.resource_registry.update(view)
        return True

    def delete_view(self, view_id=''):
        _, assocs = self.clients.resource_registry.find_objects_mult(subjects=[view_id])
        for assoc in assocs:
            self.clients.resource_registry.delete_association(assoc._id)
        self.clients.resource_registry.delete(view_id)
        return True
class DiscoveryService(BaseDiscoveryService):
    SEARCH_BUFFER_SIZE=CFG.get_safe('service.discovery.search_buffer_size', 1048576)
    MAX_SEARCH_RESULTS=CFG.get_safe('service.discovery.max_search_results', 250)

    """
    class docstring
    """

    def on_start(self): # pragma no cover
        super(DiscoveryService,self).on_start()

        self.use_es = CFG.get_safe('system.elasticsearch', False)

        self.elasticsearch_host = CFG.get_safe('server.elasticsearch.host','localhost')
        self.elasticsearch_port = CFG.get_safe('server.elasticsearch.port','9200')

        self.ep = EventPublisher(event_type = 'SearchBufferExceededEvent')
        self.heuristic_cutoff = 4

        self.cfg_datastore = CFG.get_safe('container.datastore.default_server', "couchdb")
        self.ds_discovery = None
        if self.cfg_datastore != "couchdb":
            self.ds_discovery = DatastoreDiscovery(self)

   
    @staticmethod
    def es_cleanup():
        es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
        es_port = CFG.get_safe('server.elasticsearch.port', '9200')
        es = ep.ElasticSearch(
            host=es_host,
            port=es_port,
            timeout=10
        )
        indexes = STD_INDEXES.keys()
        indexes.append('%s_resources_index' % get_sys_name().lower())
        indexes.append('%s_events_index' % get_sys_name().lower())

        for index in indexes:
            IndexManagementService._es_call(es.river_couchdb_delete,index)
            IndexManagementService._es_call(es.index_delete,index)

    #===================================================================
    # Views
    #===================================================================
        

    def create_view(self, view_name='', description='', fields=None, order=None, filters=''):
        """Creates a view which has the specified search fields, the order in which the search fields are presented
        to a query and a term filter.
        @param view_name Name of the view
        @param description Simple descriptive sentence
        @param fields Search fields
        @param order List of fields to determine order of precendence in which the results are presented
        @param filter Simple term filter

        @param view_name    str
        @param description    str
        @param fields    list
        @param order    list
        @param filters    str
        @retval view_id    str
        """
        res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
        if len(res) > 0:
            raise BadRequest('The view resource with name: %s, already exists.' % view_name)

        #======================
        # Arg Validations
        #======================
        validate_is_instance(fields,list, 'Specified fields must be a list.')
        validate_true(len(fields)>0, 'Specfied fields must be a list.')
        if order is not None:
            validate_is_instance(order,list, 'Specified order must be a list of fields')
            for field in order:
                if not field in fields:
                    raise BadRequest('The specified ordering field was not part of the search fields.')

        fields = set(fields) # Convert fields to a set for aggregation across the catalogs
        #======================================================================================================
        # Priorty Queue Index Matching
        #======================================================================================================

        pq = [] # Priority queue for matching
        catalog_id = None
        catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
        for catalog in catalogs:
            if set(catalog.catalog_fields).issubset(fields):
                index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
                heapq.heappush(pq, (index_num,catalog))
        if pq:
            weight, catalog = heapq.heappop(pq)
            if weight < self.heuristic_cutoff:
                catalog_id = catalog._id

                
        if catalog_id is None:
            catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))

        view_res = View(name=view_name, description=description)
        view_res.order = order
        view_res.filters = filters
        view_id, _ = self.clients.resource_registry.create(view_res)
        self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
        return view_id

    def read_view(self, view_id=''):
        return self.clients.resource_registry.read(view_id)

    def update_view(self, view=None):
        self.clients.resource_registry.update(view)
        return True

    def delete_view(self, view_id=''):
        _, assocs = self.clients.resource_registry.find_objects_mult(subjects=[view_id])
        for assoc in assocs:
            self.clients.resource_registry.delete_association(assoc._id)
        self.clients.resource_registry.delete(view_id)
        return True

    def list_catalogs(self, view_id=''):
        catalogs, _ = self.clients.resource_registry.find_objects(subject=view_id, object_type=RT.Catalog, predicate=PRED.hasCatalog, id_only=True)
        return catalogs


    #===================================================================
    # Helper Methods
    #===================================================================
    def _match_query_sources(self, source_name):
        index = self.clients.index_management.find_indexes(source_name)
        if index:
            return index
        _, resources = self.clients.resource_registry.find_resources(name=source_name, id_only=True)
        for res in resources:
            t = res['type']
            if t == 'View' or t == 'ElasticSearchIndex' or t == 'Catalog':
                return res['id']
        return None


    #===================================================================
    # Query Methods
    #===================================================================


    def query(self, query=None, id_only=True):
        validate_true(query,'Invalid query')

        return self.request(query, id_only)


    def query_couch(self, index_id='', key='', limit=0, offset=0, id_only=True):
        raise BadRequest('Not Implemented Yet')
#        cc = self.container
#
#        datastore_name = source.datastore_name
#        db = cc.datastore_manager.get_datastore(datastore_name)
#        view_name = source.view_name
#        opts = DotDict(include_docs=True)
#        opts.start_key = [query.query]
#        opts.end_key = [query.query,{}]
#        if query.results:
#            opts.limit = query.results
#        if query.offset:
#            opts.skip = query.offset
#
#        return db.query_view(view_name,opts=opts)

    def traverse(self, resource_id=''):
        """Breadth-first traversal of the association graph for a specified resource.

        @param resource_id    str
        @retval resources    list
        """
        
        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_objects_mult(subjects=resource_ids,id_only=True)[0]
            
        visited_resources = deque(edges([resource_id]))
        traversal_queue = deque()
        done = False
        t = None
        while not done:
            t = traversal_queue or deque(visited_resources)
            traversal_queue = deque()
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    traversal_queue.append(e)
            if not len(traversal_queue): done = True



        return list(visited_resources)

    def reverse_traverse(self, resource_id=''):
        """Breadth-first traversal of the association graph for a specified resource.

        @param resource_id    str
        @retval resources    list
        """
        
        def edges(resource_ids=[]):
            if not isinstance(resource_ids,list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_subjects_mult(objects=resource_ids,id_only=True)[0]
            
        visited_resources = deque(edges([resource_id]))
        traversal_queue = deque()
        done = False
        t = None
        while not done:
            t = traversal_queue or deque(visited_resources)
            traversal_queue = deque()
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    traversal_queue.append(e)
            if not len(traversal_queue): done = True



        return list(visited_resources)
    

    def iterative_traverse(self, resource_id='', limit=-1):
        '''
        Iterative breadth first traversal of the resource associations
        '''
        #--------------------------------------------------------------------------------
        # Retrieve edges for this resource
        #--------------------------------------------------------------------------------
        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_objects_mult(subjects=resource_ids,id_only=True)[0]
            
        gathered = deque()
        visited_resources = deque(edges([resource_id]))
        while limit>0:
            t = gathered or deque(visited_resources)
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    gathered.append(e)
            if not len(gathered): break

            t = deque(gathered)
            gathered = deque()
            limit -= 1

        return list(visited_resources)

    def iterative_reverse_traverse(self, resource_id='', limit=-1):
        '''
        Iterative breadth first traversal of the resource associations
        '''
        #--------------------------------------------------------------------------------
        # Retrieve edges for this resource
        #--------------------------------------------------------------------------------
        def edges(resource_ids=[]):
            if not isinstance(resource_ids, list):
                resource_ids = list(resource_ids)
            return self.clients.resource_registry.find_subjects_mult(objects=resource_ids,id_only=True)[0]
            
        gathered = deque()
        visited_resources = deque(edges([resource_id]))
        while limit>0:
            t = gathered or deque(visited_resources)
            for e in edges(t):
                if not e in visited_resources:
                    visited_resources.append(e)
                    gathered.append(e)
            if not len(gathered): break

            t = deque(gathered)
            gathered = deque()
            limit -= 1

        return list(visited_resources)


            


    def intersect(self, left=[], right=[]):
        """The intersection between two sets of resources.

        @param left    list
        @param right    list
        @retval result    list
        """
        return list(set(left).intersection(right))

    def union(self, left=[], right=[]):
        return list(set(left).union(right))

    def parse(self, search_request='', id_only=True):
        parser = QueryLanguage()
        query_request = parser.parse(search_request)
        return self.request(query_request, id_only=id_only)

    def query_request(self, query=None, limit=0, id_only=False):
        validate_is_instance(query,dict, 'invalid query')

        #---------------------------------------------
        # Term Search
        #---------------------------------------------
        if QueryLanguage.query_is_term_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id= source_id,
                field    = query['field'],
                value    = query['value'],
                limit    = limit,
                id_only  = id_only
            )
            
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']

            return self.query_term(**kwargs)

        #---------------------------------------------
        # Fuzzy searching (phrases and such)
        #---------------------------------------------
        elif QueryLanguage.query_is_fuzzy_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id= source_id,
                fuzzy    = True,
                field    = query['field'],
                value    = query['fuzzy'],
                limit    = limit,
                id_only  = id_only
            )
            
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']

            return self.query_term(**kwargs)
        
        #---------------------------------------------
        # Match searching (phrases and such)
        #---------------------------------------------
        elif QueryLanguage.query_is_match_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id= source_id,
                match    = True,
                field    = query['field'],
                value    = query['match'],
                limit    = limit,
                id_only  = id_only
            )
            
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']

            return self.query_term(**kwargs)
        
        
        #---------------------------------------------
        # Association Search
        #---------------------------------------------
        elif QueryLanguage.query_is_association_search(query):
            kwargs = dict(
                resource_id = query['association'],
                id_only     = id_only
            )
            if query.get('depth'):
                kwargs['depth'] = query['depth']
            return self.query_association(**kwargs)

        elif QueryLanguage.query_is_owner_search(query):
            kwargs = dict(
                resource_id = query['owner'],
                id_only     = id_only
            )
            if query.get('depth'):
                kwargs['depth'] = query['depth']
            return self.query_owner(**kwargs)
        
        #---------------------------------------------
        # Range Search
        #---------------------------------------------
        elif QueryLanguage.query_is_range_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id  = source_id,
                field      = query['field'],
                limit      = limit,
                id_only    = id_only
            )
            if get_safe(query,'range.from') is not None:
                kwargs['from_value'] = query['range']['from']
            if get_safe(query,'range.to') is not None:
                kwargs['to_value'] = query['range']['to']
                
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            
            return self.query_range(**kwargs)
        
        #---------------------------------------------
        # Time Search
        #---------------------------------------------
        elif QueryLanguage.query_is_time_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id  = source_id,
                field      = query['field'],
                limit      = limit,
                id_only    = id_only
            )
            if get_safe(query,'time.from') is not None:
                kwargs['from_value'] = query['time']['from']
            if get_safe(query,'time.to') is not None:
                kwargs['to_value'] = query['time']['to']
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            
            return self.query_time(**kwargs)
        
        
        #---------------------------------------------
        # Time Bounds Search
        #---------------------------------------------

        elif QueryLanguage.query_is_time_bounds_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id  = source_id,
                field      = query['field'],
                limit      = limit,
                id_only    = id_only
            )
            if get_safe(query,'time_bounds.from') is not None:
                kwargs['from_value'] = query['time_bounds']['from']
            if get_safe(query,'time_bounds.to') is not None:
                kwargs['to_value'] = query['time_bounds']['to']
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            
            return self.query_time_bounds(**kwargs)
        
        #---------------------------------------------
        # Vertical Bounds Search
        #---------------------------------------------

        elif QueryLanguage.query_is_vertical_bounds_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id  = source_id,
                field      = query['field'],
                limit      = limit,
                id_only    = id_only
            )
            if get_safe(query,'vertical_bounds.from') is not None:
                kwargs['from_value'] = query['vertical_bounds']['from']
            if get_safe(query,'vertical_bounds.to') is not None:
                kwargs['to_value'] = query['vertical_bounds']['to']
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            
            return self.query_vertical_bounds(**kwargs)
        
        #---------------------------------------------
        # Collection Search
        #---------------------------------------------
        elif QueryLanguage.query_is_collection_search(query):
            return self.query_collection(
                collection_id = query['collection'],
                id_only       = id_only
            )
        
        #---------------------------------------------
        # Geo Distance Search
        #---------------------------------------------
        elif QueryLanguage.query_is_geo_distance_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id = source_id,
                field     = query['field'],
                origin    = [query['lon'], query['lat']],
                distance  = query['dist'],
                units     = query['units'],
                id_only   = id_only
            )
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            return self.query_geo_distance(**kwargs)
        
        #---------------------------------------------
        # Geo Bounding Box Search
        #---------------------------------------------
        elif QueryLanguage.query_is_geo_bbox_search(query):
            source_id = self._match_query_sources(query['index']) or query['index']
            kwargs = dict(
                source_id    = source_id,
                field        = query['field'],
                top_left     = query['top_left'],
                bottom_right = query['bottom_right'],
                limit        = limit,
                id_only      = id_only,
            )
            if query.get('limit'):
                kwargs['limit'] = query['limit']
            if query.get('order'):
                kwargs['order'] = query['order']
            if query.get('offset'):
                kwargs['offset'] = query['offset']
            return self.query_geo_bbox(**kwargs)


        #@todo: query for couch
        raise BadRequest('improper query: %s' % query)

    def _multi(self, cb,source, *args, **kwargs):
        '''
        Manage the different collections of indexes for queries, views, catalogs
        Expand the resource into it's components and call the callback for each subcategory
        '''
        if isinstance(source, View):
            catalogs = self.list_catalogs(source._id)
            result_queue = list()
            for catalog in catalogs:
                result_queue.extend(cb(catalog, *args, **kwargs))
            if kwargs.has_key('limit') and kwargs['limit']:
                return result_queue[:kwargs['limit']]
            return result_queue

        if isinstance(source, Catalog):
            indexes = self.clients.catalog_management.list_indexes(source._id, id_only=True)
            result_queue = list()
            for index in indexes:
                result_queue.extend(cb(index, *args, **kwargs))
            if kwargs.has_key('limit') and kwargs['limit']:
                return result_queue[:kwargs['limit']]
            return result_queue
        return None

    def query_term(self, source_id='', field='', value='', fuzzy=False, match=False, order=None, limit=0, offset=0, id_only=False):
        '''
        Elasticsearch Query against an index
        > discovery.query_index('indexID', 'name', '*', order={'name':'asc'}, limit=20, id_only=False)
        '''
        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable system.elasticsearch to make queries.')

        validate_true(source_id, 'Unspecified source_id')
        validate_true(field, 'Unspecified field')
        validate_true(value, 'Unspecified value')


        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        # If source is a view, catalog or collection go through it and recursively call query_range on all the results in the indexes
        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        iterate = self._multi(self.query_term, source, field=field, value=value, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate


        index = source
        validate_is_instance(index, ElasticSearchIndex, '%s does not refer to a valid index.' % index)
        if order: 
            validate_is_instance(order,dict, 'Order is incorrect.')
            es.sort(**order)

        if limit:
            es.size(limit)

        if offset:
            es.from_offset(offset)

        if field == '*':
            field = '_all'

        if fuzzy:
            query = ep.ElasticQuery.fuzzy_like_this(value, fields=[field])
        elif match:
            match_query = ep.ElasticQuery.match(field=field,query=value)
            query = {"match_phrase_prefix":match_query['match']}
            
        elif '*' in value:
            query = ep.ElasticQuery.wildcard(field=field, value=value)
        else:
            query = ep.ElasticQuery.field(field=field, query=value)

        response = IndexManagementService._es_call(es.search_index_advanced,index.index_name,query)

        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)

    def query_range(self, source_id='', field='', from_value=None, to_value=None, order=None, limit=0, offset=0, id_only=False):
        
        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable in res/config/pyon.yml')

        if from_value is not None:
            validate_true(isinstance(from_value,int) or isinstance(from_value,float), 'from_value is not a valid number')
        if to_value is not None:
            validate_true(isinstance(to_value,int) or isinstance(to_value,float), 'to_value is not a valid number')
        validate_true(source_id, 'source_id not specified')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)


        source = self.clients.resource_registry.read(source_id)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        # If source is a view, catalog or collection go through it and recursively call query_range on all the results in the indexes
        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        iterate = self._multi(self.query_range, source, field=field, from_value=from_value, to_value=to_value, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex,'%s does not refer to a valid index.' % source_id)
        if order:
            validate_is_instance(order,dict,'Order is incorrect.')
            es.sort(**order)

        if limit:
            es.size(limit)

        if field == '*':
            field = '_all'

        query = ep.ElasticQuery.range(
            field      = field,
            from_value = from_value,
            to_value   = to_value
        )
        response = IndexManagementService._es_call(es.search_index_advanced,index.index_name,query)

        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)

    def query_time(self, source_id='', field='', from_value=None, to_value=None, order=None, limit=0, offset=0, id_only=False):
        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable in res/config/pyon.yml')

        if from_value is not None:
            validate_is_instance(from_value,basestring,'"From" is not a valid string (%s)' % from_value)

        if to_value is not None:
            validate_is_instance(to_value,basestring,'"To" is not a valid string')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        # If source is a view, catalog or collection go through it and recursively call query_time on all the results in the indexes
        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        iterate = self._multi(self.query_time, source, field=field, from_value=from_value, to_value=to_value, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex,'%s does not refer to a valid index.' % source_id)
        if order:
            validate_is_instance(order,dict,'Order is incorrect.')
            es.sort(**order)

        if limit:
            es.size(limit)

        if field == '*':
            field = '_all'

        if from_value is not None:
            from_value = calendar.timegm(dateutil.parser.parse(from_value).timetuple()) * 1000

        if to_value is not None:
            to_value = calendar.timegm(dateutil.parser.parse(to_value).timetuple()) * 1000

        query = ep.ElasticQuery.range(
            field      = field,
            from_value = from_value,
            to_value   = to_value
        )

        response = IndexManagementService._es_call(es.search_index_advanced,index.index_name,query)

        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)


    def query_time_bounds(self, source_id='', field='', from_value=None, to_value=None, order=None, limit=0, offset=0, id_only=False):
        if from_value is not None:
            validate_is_instance(from_value,basestring,'"From" is not a valid string (%s)' % from_value)

        if to_value is not None:
            validate_is_instance(to_value,basestring,'"To" is not a valid string')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        # If source is a view, catalog or collection go through it and recursively call query_time on all the results in the indexes
        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        iterate = self._multi(self.query_time, source, field=field, from_value=from_value, to_value=to_value, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex,'%s does not refer to a valid index.' % source_id)
        if order:
            validate_is_instance(order,dict,'Order is incorrect.')
            es.sort(**order)

        if field == '*':
            field = '_all'
            start_time = 'start_datetime'
            end_time = 'end_datetime'
        else:
            start_time = '%s.start_datetime' % field
            end_time = '%s.end_datetime' % field



        if from_value is not None:
            from_value = calendar.timegm(dateutil.parser.parse(from_value).timetuple()) * 1000

        if to_value is not None:
            to_value = calendar.timegm(dateutil.parser.parse(to_value).timetuple()) * 1000

        query = {
          "query": {
            "match_all": {}
          },
          "filter": {
            "and": [
              {
                "or": [
                  {
                    "range": {
                      start_time: {
                        "gte": from_value
                      }
                    }
                  },
                  {
                    "range": {
                      end_time: {
                        "gte": from_value
                      }
                    }
                  }
                ]
              },
              {
                "or": [
                  {
                    "range": {
                      start_time: {
                        "lte": to_value
                      }
                    }
                  },
                  {
                    "range": {
                      end_time: {
                        "lte": to_value
                      }
                    }
                  }
                ]
              }
            ]
          }
        }
        if limit:
            query['size'] = limit
        if offset:
            query['from'] = offset

        
        response = IndexManagementService._es_call(es.raw_query,'%s/_search' % index.index_name,method='POST', data=query, host=self.elasticsearch_host, port=self.elasticsearch_port)
        IndexManagementService._check_response(response)
        return self._results_from_response(response, id_only)
 

    def query_vertical_bounds(self, source_id='', field='', from_value=None, to_value=None, order=None, limit=0, offset=0, id_only=False):
        if from_value is not None:
            validate_is_instance(from_value,float,'"From" is not a valid float (%s)' % from_value)

        if to_value is not None:
            validate_is_instance(to_value,float,'"To" is not a valid float')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        # If source is a view, catalog or collection go through it and recursively call query_time on all the results in the indexes
        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
        iterate = self._multi(self.query_time, source, field=field, from_value=from_value, to_value=to_value, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex,'%s does not refer to a valid index.' % source_id)
        if order:
            validate_is_instance(order,dict,'Order is incorrect.')
            es.sort(**order)


        if field == '*':
            field = '_all'
            vertical_min = 'geospatial_vertical_min'
            vertical_max = 'geospatial_vertical_max'
        else:
            vertical_min = '%s.geospatial_vertical_min' % field
            vertical_max = '%s.geospatial_vertical_max' % field


        query = {
          "query": {
            "match_all": {}
          },
          "filter": {
            "and": [
              {
                "or": [
                  {
                    "range": {
                      vertical_min: {
                        "gte": from_value
                      }
                    }
                  },
                  {
                    "range": {
                      vertical_max: {
                        "gte": from_value
                      }
                    }
                  }
                ]
              },
              {
                "or": [
                  {
                    "range": {
                      vertical_min: {
                        "lte": to_value
                      }
                    }
                  },
                  {
                    "range": {
                      vertical_max: {
                        "lte": to_value
                      }
                    }
                  }
                ]
              }
            ]
          }
        }
        if limit:
            query['size'] = limit
        if offset:
            query['from'] = offset

        response = IndexManagementService._es_call(es.raw_query,'%s/_search' % index.index_name,method='POST', data=query, host=self.elasticsearch_host, port=self.elasticsearch_port)
        IndexManagementService._check_response(response)
        retval= self._results_from_response(response, id_only)
        return retval


    def query_association(self,resource_id='', depth=0, id_only=False):
        validate_true(resource_id, 'Unspecified resource')
        if depth:
            resource_ids = self.iterative_traverse(resource_id, depth-1)
        else:
            resource_ids = self.traverse(resource_id)
        if id_only:
            return resource_ids

        if not isinstance(resource_ids, list):
            resource_ids = list(resource_ids)
        resources = self.clients.resource_registry.read_mult(resource_ids)

        return resources

    def query_owner(self, resource_id='', depth=0, id_only=False):
        validate_true(resource_id, 'Unspecified resource')
        if depth:
            resource_ids = self.iterative_traverse(resource_id, depth-1)
        else:
            resource_ids = self.reverse_traverse(resource_id)
        if id_only:
            return resource_ids

        if not isinstance(resource_ids, list):
            resource_ids = list(resource_ids)
        resources = self.clients.resource_registry.read_mult(resource_ids)

        return resources



    def query_collection(self,collection_id='', id_only=False):
        validate_true(collection_id, 'Unspecified collection id')
        resource_ids = self.clients.index_management.list_collection_resources(collection_id, id_only=True)
        if id_only:
            return resource_ids
        
        resources = map(self.clients.resource_registry.read,resource_ids)
        return resources

    def query_geo_distance(self, source_id='', field='', origin=None, distance='', units='mi',order=None, limit=0, offset=0, id_only=False):
        validate_true(isinstance(origin,(tuple,list)) , 'Origin is not a list or tuple.')
        validate_true(len(origin)==2, 'Origin is not of the right size: (2)')

        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable in res/config/pyon.yml')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)
        source = self.clients.resource_registry.read(source_id)

        iterate = self._multi(self.query_geo_distance, source=source, field=field, origin=origin, distance=distance) 
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex, '%s does not refer to a valid index.' % index)

        sorts = ep.ElasticSort()
        if order is not None and isinstance(order,dict):
            sort_field = order.keys()[0]
            value = order[sort_field]
            sorts.sort(sort_field,value)
            es.sorted(sorts)

        if limit:
            es.size(limit)

        if offset:
            es.from_offset(offset)

        if field == '*':
            field = '_all'


        sorts.geo_distance(field, origin, units)

        es.sorted(sorts)

        filter = ep.ElasticFilter.geo_distance(field,origin, '%s%s' %(distance,units))

        es.filtered(filter)

        query = ep.ElasticQuery.match_all()

        response = IndexManagementService._es_call(es.search_index_advanced,index.index_name,query)
        IndexManagementService._check_response(response)

        return self._results_from_response(response,id_only)


    def query_geo_bbox(self, source_id='', field='', top_left=None, bottom_right=None, order=None, limit=0, offset=0, id_only=False):
        validate_true(isinstance(top_left, (list,tuple)), 'Top Left is not a list or a tuple')
        validate_true(len(top_left)==2, 'Top Left is not of the right size: (2)')
        validate_true(isinstance(bottom_right, (list,tuple)), 'Bottom Right is not a list or a tuple')
        validate_true(len(bottom_right)==2, 'Bottom Right is not of the right size: (2)')

        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable in res/config/pyon.yml')

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)
        source = self.clients.resource_registry.read(source_id)

        iterate = self._multi(self.query_geo_bbox, source=source, field=field, top_left=top_left, bottom_right=bottom_right, order=order, limit=limit, offset=offset, id_only=id_only)
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index,ElasticSearchIndex, '%s does not refer to a valid index.' % index)

        sorts = ep.ElasticSort()
        if order is not None and isinstance(order,dict):
            sort_field = order.keys()[0]
            value = order[sort_field]
            sorts.sort(sort_field,value)
            es.sorted(sorts)

        if limit:
            es.size(limit)

        if offset:
            es.from_offset(offset)

        if field == '*':
            field = '_all'


        filter = ep.ElasticFilter.geo_bounding_box(field, top_left, bottom_right)

        es.filtered(filter)

        query = ep.ElasticQuery.match_all()

        response = IndexManagementService._es_call(es.search_index_advanced,index.index_name,query)
        IndexManagementService._check_response(response)

        return self._results_from_response(response,id_only)

        

    def es_complex_query(self,query,and_queries=None,or_queries=None):
        pass


    def es_map_query(self, query): 
        '''
        Maps an query request to an ElasticSearch query
        '''
        if not self.use_es:
            raise BadRequest('Can not make queries without ElasticSearch, enable in res/config/pyon.yml')
        if QueryLanguage.query_is_term_search(query):
            return ep.ElasticQuery.wildcard(field=query['field'],value=query['value'])
        if QueryLanguage.query_is_range_search(query):
            return ep.ElasticQuery.range(
                field      = query['field'],
                from_value = query['range']['from'],
                to_value   = query['range']['to']
            )
        

    def request(self, query=None, id_only=True):
        if not query:
            raise BadRequest('No request query provided')

        if "QUERYEXP" in query and self.ds_discovery:
            # Support for datastore queries
            pass

        else:
            if not query.has_key('query'):
                raise BadRequest('Unsuported request. %s')

            #==============================
            # Check the form of the query
            #==============================
            #@todo: convert to IonObject
            if not (query.has_key('query')):
                raise BadRequest('Improper query request: %s' % query)

        # Inject RR query execution e.g. for postgres
        if self.ds_discovery:
            res = self.ds_discovery.execute_query(query, id_only=id_only)
            return res

        # ---------------------------
        # Number of results to return
        # ---------------------------
        limit = int(query.get('limit',self.MAX_SEARCH_RESULTS))

        query_queue = list()

        query = DotDict(query)
       
        # -- former tier-1 (no and/or) search, returns an elasticsearch object
        if (len(query.get('and',[])) + len(query.get('or',[])) == 0 ):
            return self.query_request(query.query,limit=self.SEARCH_BUFFER_SIZE)[:limit]
 
        query_queue.append(self.query_request(query.query,limit=self.SEARCH_BUFFER_SIZE, id_only=True))
        
        #==================
        # Intersection
        #==================
        for q in query.get('and',[]):
            query_queue.append(self.query_request(q, limit=self.SEARCH_BUFFER_SIZE, id_only=True))
        while len(query_queue) > 1:
            tmp = self.intersect(query_queue.pop(), query_queue.pop())
            query_queue.append(tmp)
        
        #==================
        # Union
        #==================
        for q in query.get('or',[]):
            query_queue.append(self.query_request(q, limit=self.SEARCH_BUFFER_SIZE, id_only=True))
        while len(query_queue) > 1:
            tmp = self.union(query_queue.pop(), query_queue.pop())
            query_queue.append(tmp)
        
        if id_only:
            return query_queue[0][:limit]

        objects = self.clients.resource_registry.read_mult(query_queue[0][:limit])
        return objects




    def raise_search_buffer_exceeded(self):
        self.ep.publish_event(origin='Discovery Service', description='Search buffer was exceeded, results may not contain all the possible results.')

    def _results_from_response(self, response, id_only):
        deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        if not (response.has_key('hits') and response['hits'].has_key('hits')):
            return []

        hits = response['hits']['hits']
       
        if len(hits) > 0:
            if len(hits) >= self.SEARCH_BUFFER_SIZE:
                log.warning("Query results exceeded search buffer limitations")
                self.raise_search_buffer_exceeded()
            if id_only:
                return [str(i['_id']) for i in hits]
            results = map(deserializer.deserialize,hits)
            return results
        
        else:
            return []
class DiscoveryService(BaseDiscoveryService):
    MAX_SEARCH_RESULTS = CFG.get_safe('service.discovery.max_search_results', 250)

    def on_start(self):
        super(DiscoveryService, self).on_start()

        cfg_datastore = CFG.get_safe('container.datastore.default_server')
        if cfg_datastore != "postgresql":
            raise Exception("Discovery service does not support datastores other than postgresql")

        self.ds_discovery = DatastoreDiscovery(self)

   
    #===================================================================
    # Query Methods
    #===================================================================

    def parse(self, search_request='', id_only=True, search_args=None):
        """Parses a given string request and assembles the query, processes the query and returns the results of the query.
        See the query language definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format

        @param search_request    str
        @param id_only    bool
        @param search_args dict
        @retval results    list
        """
        log.info("Search DSL: %s", search_request)
        query_request = self._parse_query_string(search_request)
        return self._discovery_request(query_request, id_only=id_only, search_args=search_args, query_params=search_args)

    def _parse_query_string(self, query_string):
        """Given a query string in Discovery service DSL, parse and return query structure"""
        parser = QueryLanguage()
        query_request = parser.parse(query_string)
        return query_request

    def query(self, query=None, id_only=True, search_args=None):
        """Issue a query provided in structured dict format or internal datastore query format.
        Returns a list of resource or event objects or their IDs only.
        Search_args may contain parameterized values.
        See the query format definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format

        @param query    dict
        @param id_only    bool
        @param search_args dict
        @retval results    list
        """
        validate_true(query, 'Invalid query')

        return self._discovery_request(query, id_only, search_args=search_args, query_params=search_args)

    def query_view(self, view_id='', view_name='', ext_query=None, id_only=True, search_args=None):
        """Execute an existing query as defined within a View resource, providing additional arguments for
        parameterized values.
        If ext_query is provided, it will be combined with the query defined by the View.
        Search_args may contain parameterized values.
        Returns a list of resource or event objects or their IDs only.
        """
        if not view_id and not view_name:
            raise BadRequest("Must provide argument view_id or view_name")
        if view_id and view_name:
            raise BadRequest("Cannot provide both arguments view_id and view_name")
        if view_id:
            view_obj = self.clients.resource_registry.read(view_id)
        else:
            view_obj = self.ds_discovery.get_builtin_view(view_name)
            if not view_obj:
                view_objs, _ = self.clients.resource_registry.find_resources(restype=RT.View, name=view_name)
                if not view_objs:
                    raise NotFound("View with name '%s' not found" % view_name)
                view_obj = view_objs[0]

        if view_obj.type_ != RT.View:
            raise BadRequest("Argument view_id is not a View resource")
        view_query = view_obj.view_definition
        if not QUERY_EXP_KEY in view_query:
            raise BadRequest("Unknown View query format")

        # Get default query params and override them with provided args
        param_defaults = {param.name: param.default for param in view_obj.view_parameters}
        query_params = param_defaults
        if view_obj.param_values:
            query_params.update(view_obj.param_values)
        if search_args:
            query_params.update(search_args)

        # Merge ext_query into query
        if ext_query:
            if ext_query["where"] and view_query["where"]:
                view_query["where"] = [DQ.EXP_AND, [view_query["where"], ext_query["where"]]]
            else:
                view_query["where"] = view_query["where"] or ext_query["where"]
            if ext_query["order_by"]:
                # Override ordering if present
                view_query["where"] = ext_query["order_by"]

            # Other query settings
            view_qargs = view_query["query_args"]
            ext_qargs = ext_query["query_args"]
            view_qargs["id_only"] = ext_qargs.get("id_only", view_qargs["id_only"])
            view_qargs["limit"] = ext_qargs.get("limit", view_qargs["limit"])
            view_qargs["skip"] = ext_qargs.get("skip", view_qargs["skip"])

        return self._discovery_request(view_query, id_only=id_only,
                                       search_args=search_args, query_params=query_params)

    def _discovery_request(self, query=None, id_only=True, search_args=None, query_params=None):
        search_args = search_args or {}
        if not query:
            raise BadRequest('No request query provided')

        if QUERY_EXP_KEY in query and self.ds_discovery:
            query.setdefault("query_args", {})["id_only"] = id_only
            # Query in datastore query format (dict)
            log.debug("Executing datastore query: %s", query)

        elif "QUERYDSL" in query:
            # Query in DSL format
            if "query_str" not in query:
                raise BadRequest('No query_str provided')
            query = self._parse_query_string(query["query_str"])

        elif 'query' not in query:
            raise BadRequest('Unsupported request. %s' % query)

        # if count requested, run id_only query without limit/skip
        count = search_args.get("count", False)
        if count:
            # Only return the count of ID only search
            query.pop("limit", None)
            query.pop("skip", None)
            res = self.ds_discovery.execute_query(query, id_only=True, query_args=search_args, query_params=query_params)
            return [len(res)]

        # TODO: Not all queries are permissible by all users

        # Execute the query
        query_results = self.ds_discovery.execute_query(query, id_only=id_only,
                                                        query_args=search_args, query_params=query_params)

        # Strip out unwanted object attributes for size
        filtered_res = self._strip_query_results(query_results, id_only=id_only, search_args=search_args)

        return filtered_res

    def _strip_query_results(self, query_results, id_only, search_args):
        # Filter the results for smaller result size
        attr_filter = search_args.get("attribute_filter", [])
        if type(attr_filter) not in (list, tuple):
            raise BadRequest("Illegal argument type: attribute_filter")

        if not id_only and attr_filter:
            filtered_res = [dict(__noion__=True, **{k: v for k, v in obj.__dict__.iteritems() if k in attr_filter or k in {"_id", "type_"}}) for obj in query_results]
            return filtered_res
        return query_results


    #===================================================================
    # View Management
    #===================================================================

    def create_view(self, view=None):
        if view is None or not isinstance(view, View):
            raise BadRequest("Illegal argument: view")

        # view_objs, _ = self.clients.resource_registry.find_resources(restype=RT.View, name=view.name)
        # if view_objs:
        #     raise BadRequest("View with name '%s' already exists" % view.name)

        view_id, _ = self.clients.resource_registry.create(view)
        return view_id

    def read_view(self, view_id=''):
        view_res = self.clients.resource_registry.read(view_id)
        if not isinstance(view_res, View):
            raise BadRequest("Resource %s is not a View" % view_id)
        return view_res

    def update_view(self, view=None):
        if view is None or not isinstance(view, View):
            raise BadRequest("Illegal argument: view")
        self.clients.resource_registry.update(view)
        return True

    def delete_view(self, view_id=''):
        self.clients.resource_registry.delete(view_id)
        return True


    def create_catalog_view(self, view_name='', description='', fields=None, order=None, filters=''):
        """Creates a view which has the specified search fields, the order in which the search fields are presented
        to a query and a term filter.
        @param view_name Name of the view
        @param description Simple descriptive sentence
        @param fields Search fields
        @param order List of fields to determine order of precendence in which the results are presented
        @param filters Simple term filter
        """
        res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
        if len(res) > 0:
            raise BadRequest('The view resource with name: %s, already exists.' % view_name)

        #======================
        # Arg Validations
        #======================
        validate_is_instance(fields,list, 'Specified fields must be a list.')
        validate_true(len(fields)>0, 'Specfied fields must be a list.')
        if order is not None:
            validate_is_instance(order,list, 'Specified order must be a list of fields')
            for field in order:
                if not field in fields:
                    raise BadRequest('The specified ordering field was not part of the search fields.')

        fields = set(fields) # Convert fields to a set for aggregation across the catalogs
        #======================================================================================================
        # Priorty Queue Index Matching
        #======================================================================================================

        pq = [] # Priority queue for matching
        catalog_id = None
        catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
        for catalog in catalogs:
            if set(catalog.catalog_fields).issubset(fields):
                index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
                heapq.heappush(pq, (index_num,catalog))
        if pq:
            weight, catalog = heapq.heappop(pq)
            if weight < 4:
                catalog_id = catalog._id


        if catalog_id is None:
            catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))

        view_res = View(name=view_name, description=description)
        view_res.order = order
        view_res.filters = filters
        view_id, _ = self.clients.resource_registry.create(view_res)
        self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
        return view_id