def index_fields_dynamic(request): result = {'status': -1, 'message': 'Error'} try: name = request.POST['name'] dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name) result['message'] = '' result['fields'] = [ Collection2._make_field(name, properties) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['gridlayout_header_fields'] = [ Collection2._make_gridlayout_header_field( { 'name': name, 'type': properties.get('type') }, True) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['status'] = 0 except Exception, e: result['message'] = force_unicode(e)
def browse(request, name, is_mobile=False): collections = SearchController(request.user).get_all_indexes() if not collections: return no_collections(request) collection = Collection2(user=request.user, name=name) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} template = 'search.mako' if is_mobile: template = 'search_m.mako' return render(template, request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'autoLoad': True, 'collections': collections, 'layout': [ {"size":12,"rows":[{"widgets":[ {"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget", "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}], "drops":["temp"],"klass":"card card-home card-column span10"} ], 'is_latest': LATEST.get() }), 'is_owner': True, 'can_edit_index': can_edit_index(request.user), 'mobile': is_mobile })
def new_search(request, is_embeddable=False): collections = SearchController(request.user).get_all_indexes() if not collections: return no_collections(request) collection = Collection2(user=request.user, name=collections[0]) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} template = 'search.mako' if is_embeddable: template = 'search_embeddable.mako' return render(template, request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'collections': collections, 'layout': [ {"size":2,"rows":[{"widgets":[]}],"drops":["temp"],"klass":"card card-home card-column span2"}, {"size":10,"rows":[{"widgets":[ {"size":12,"name":"Filter Bar","widgetType":"filter-widget", "id":"99923aef-b233-9420-96c6-15d48293532b", "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}, {"widgets":[ {"size":12,"name":"Grid Results","widgetType":"resultset-widget", "id":"14023aef-b233-9420-96c6-15d48293532b", "properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}], "drops":["temp"],"klass":"card card-home card-column span10"}, ], 'is_latest': LATEST.get(), }), 'is_owner': True, 'can_edit_index': can_edit_index(request.user) })
def copy_collections(self, collection_ids): result = {'status': -1, 'message': ''} try: for doc2 in self.get_shared_search_collections(): if doc2.id in collection_ids: name = doc2.name + '-copy' doc2.pk = None doc2.id = None doc2.uuid = str(uuid.uuid4()) doc2.name = name doc2.owner = self.user doc2.save() copy_doc = Document.objects.link(doc2, owner=copy.owner, name=copy.name, description=copy.description) copy = Collection2(self.user, document=doc2) copy.data['collection']['label'] = name doc2.update_data({'collection': copy.data['collection']}) doc2.save() result['status'] = 0 except Exception, e: LOG.exception('Error copying collection') result['message'] = unicode(str(e), "utf8")
def copy_collections(self, collection_ids): result = {'status': -1, 'message': ''} try: for doc2 in self.get_shared_search_collections(): if doc2.id in collection_ids: name = doc2.name + '-copy' copy_doc = doc2.doc.get().copy(name=name, owner=self.user) doc2.pk = None doc2.id = None doc2.uuid = str(uuid.uuid4()) doc2.name = name doc2.owner = self.user doc2.save() doc2.doc.all().delete() doc2.doc.add(copy_doc) doc2.save() copy = Collection2(self.user, document=doc2) copy.data['collection']['label'] = name doc2.update_data({'collection': copy.data['collection']}) doc2.save() result['status'] = 0 except Exception, e: print e LOG.warn('Error copying collection: %s' % e) result['message'] = unicode(str(e), "utf8")
def new_search(request): collections = SearchController(request.user).get_all_indexes() if not collections: return no_collections(request) collection = Collection2(user=request.user, name=collections[0]) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} return render( 'search.mako', request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'collections': collections, 'layout': [ { "size": 2, "rows": [{ "widgets": [] }], "drops": ["temp"], "klass": "card card-home card-column span2" }, { "size": 10, "rows": [{ "widgets": [{ "size": 12, "name": "Filter Bar", "widgetType": "filter-widget", "properties": {}, "offset": 0, "isLoading": True, "klass": "card card-widget span12" }] }, { "widgets": [{ "size": 12, "name": "Grid Results", "widgetType": "resultset-widget", "properties": {}, "offset": 0, "isLoading": True, "klass": "card card-widget span12" }] }], "drops": ["temp"], "klass": "card card-home card-column span10" }, ] }), 'is_owner': True })
def index_fields_dynamic(request): result = {'status': -1, 'message': 'Error'} try: name = request.POST['name'] dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name) result['message'] = '' result['fields'] = [Collection2._make_field(name, properties) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties] result['gridlayout_header_fields'] = [Collection2._make_gridlayout_header_field({'name': name}, True) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties] result['status'] = 0 except Exception, e: result['message'] = force_unicode(e)
def copy_collections(self, collection_ids): result = {'status': -1, 'message': ''} try: for collection in self.get_shared_search_collections().filter( id__in=collection_ids): doc2 = Document2.objects.get(type='search-dashboard', id=collection.id) name = doc2.name + '-copy' copy_doc = doc2.doc.get().copy(name=name, owner=self.user) doc2.pk = None doc2.id = None doc2.uuid = str(uuid.uuid4()) doc2.name = name doc2.owner = self.user doc2.save() doc2.doc.all().delete() doc2.doc.add(copy_doc) doc2.save() copy = Collection2(document=doc2) copy['collection']['label'] = name doc2.update_data({'collection': copy['collection']}) doc2.save() result['status'] = 0 except Exception, e: LOG.warn('Error copying collection: %s' % e) result['message'] = unicode(str(e), "utf8")
def test_query(self): collection = Collection2(user=self.user, name='log_analytics_demo') collection = json.loads(collection.get_json(self.user)) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} SolrApi(SOLR_URL.get(), self.user).query(collection['collection'], query)
def index_fields_dynamic(request): result = {"status": -1, "message": "Error"} try: name = request.POST["name"] dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name) result["message"] = "" result["fields"] = [ Collection2._make_field(name, properties) for name, properties in dynamic_fields["fields"].iteritems() if "dynamicBase" in properties ] result["gridlayout_header_fields"] = [ Collection2._make_gridlayout_header_field({"name": name, "type": properties.get("type")}, True) for name, properties in dynamic_fields["fields"].iteritems() if "dynamicBase" in properties ] result["status"] = 0 except Exception, e: result["message"] = force_unicode(e)
def index(request): hue_collections = SearchController(request.user).get_search_collections() collection_id = request.GET.get('collection') if not hue_collections or not collection_id: return admin_collections(request, True) try: collection_doc = Document2.objects.get(id=collection_id) collection_doc.doc.get().can_read_or_exception(request.user) collection = Collection2(request.user, document=collection_doc) except Exception, e: raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def get_collection(request): result = {'status': -1, 'message': ''} try: name = request.POST['name'] collection = Collection2(request.user, name=name) collection_json = collection.get_json(request.user) result['collection'] = json.loads(collection_json) result['status'] = 0 except Exception, e: result['message'] = force_unicode(e)
def get_fields(self, collection_or_core_name): api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) try: field_data = api.fields(collection_or_core_name) fields = self._format_flags(field_data['schema']['fields']) except Exception, e: LOG.warn('/luke call did not succeed: %s' % e) try: fields = api.schema_fields(collection_or_core_name) fields = Collection2._make_luke_from_schema_fields(fields) except: LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name) raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name)
def get_fields(self, collection_or_core_name): api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) try: field_data = api.fields(collection_or_core_name) fields = self._format_flags(field_data['schema']['fields']) except Exception, e: LOG.warn('/luke call did not succeed: %s' % e) try: fields = api.schema_fields(collection_or_core_name) fields = Collection2._make_luke_from_schema_fields(fields) except: LOG.exception( _('Could not fetch fields for collection %s.') % collection_or_core_name) raise PopupException( _('Could not fetch fields for collection %s. See logs for more info.' ) % collection_or_core_name)
def copy_collections(self, collection_ids): result = {'status': -1, 'message': ''} try: for doc2 in self.get_shared_search_collections(): if doc2.id in collection_ids: doc2 = Document2.objects.get(uuid=doc2.uuid) doc = doc2.doc.get() name = doc2.name + '-copy' doc2 = doc2.copy(name=name, owner=self.user) doc.copy(content_object=doc2, name=name, owner=self.user) collection = Collection2(self.user, document=doc2) collection.data['collection']['label'] = name doc2.update_data({'collection': collection.data['collection']}) doc2.save() result['status'] = 0 except Exception, e: LOG.exception('Error copying collection') result['message'] = unicode(str(e), "utf8")
def luke(self, collection): fields = self.schema_fields(collection) return {'fields': Collection2._make_luke_from_schema_fields(fields)}
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) params = self._get_params() + ( ('q', self._get_q(query)), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) json_facets = {} timeFilter = self._get_range_borders(collection, query) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']),) elif facet['type'] == 'range' or facet['type'] == 'range-up': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], 'mincount': int(facet['properties']['mincount']) } if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'): keys.update(self._get_time_filter_query(timeFilter, facet)) params += ( ('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'field': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0), 'mincount': int(facet['properties']['mincount']) } params += ( ('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'nested': _f = { 'field': facet['field'], 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'text-facet-widget' else 0), 'mincount': int(facet['properties']['mincount']), 'sort': {'count': facet['properties']['sort']}, } if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'): _f['domain'] = {} if facet['properties']['domain'].get('blockParent'): _f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent']) if facet['properties']['domain'].get('blockChildren'): _f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren']) if 'start' in facet['properties'] and not facet['properties'].get('type') == 'field': _f.update({ 'type': 'range', 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], }) if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'bucket-widget'): _f.update(self._get_time_filter_query(timeFilter, facet)) else: _f.update({ 'type': 'terms', 'field': facet['field'], 'excludeTags': facet['id'], 'offset': 0, 'numBuckets': True, 'allBuckets': True, #'prefix': '' # Forbidden on numeric fields }) if facet['properties']['canRange'] and not facet['properties']['isDate']: del _f['mincount'] # Numeric fields do not support if facet['properties']['facets']: self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1) if facet['widgetType'] == 'text-facet-widget': _fname = _f['facet'].keys()[0] _f['sort'] = {_fname: facet['properties']['sort']} # domain = '-d2:NaN' # Solr 6.4 json_facets[facet['id']] = _f elif facet['type'] == 'function': json_facets[facet['id']] = self._get_aggregate_function(facet) json_facets['processEmpty'] = True elif facet['type'] == 'pivot': if facet['properties']['facets'] or facet['widgetType'] == 'map-widget': fields = facet['field'] fields_limits = [] for f in facet['properties']['facets']: fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit'])) fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount'])) fields += ',' + f['field'] keys = { 'id': '%(id)s' % facet, 'key': '%(field)s-%(id)s' % facet, 'field': facet['field'], 'fields': fields, 'limit': int(facet['properties'].get('limit', 10)), 'mincount': int(facet['properties']['mincount']), 'fields_limits': ' '.join(fields_limits) } params += ( ('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys), ) if json_facets: params += ( ('json.facet', json.dumps(json_facets)), ) params += self._get_fq(collection, query) from search.models import Collection2 fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection)))) nested_fields = self._get_nested_fields(collection) if nested_fields: fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields))) params += (('fl', fl),) params += ( ('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 5), ('hl.fragsize', 1000), ) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append('%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += ( ('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def query(self, dashboard, query, facet=None): database, table = self._get_database_table_names(dashboard['name']) if query['qs'] == [{'q': '_root_:*'}]: return {'response': {'numFound': 0}} filters = [q['q'] for q in query['qs'] if q['q']] filters.extend(self._get_fq(dashboard, query, facet)) if facet: if facet['type'] == 'nested': fields_dimensions = [ self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet) ] last_dimension_seen = False fields = [] for f in reversed(facet['properties']['facets']): if f['aggregate']['function'] == 'count': if not last_dimension_seen: fields.insert(0, 'COUNT(*) AS Count') last_dimension_seen = True fields.insert(0, self._get_dimension_field(f)['select']) else: if not last_dimension_seen: fields.insert(0, self._get_aggregate_function(f)) if not last_dimension_seen: fields.insert(0, 'COUNT(*) as Count') fields.insert(0, self._get_dimension_field(facet)['select']) sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s GROUP BY %(fields_dimensions)s ORDER BY %(order_by)s LIMIT %(limit)s''' % { 'database': database, 'table': table, 'fields': ', '.join(fields), 'fields_dimensions': ', '.join(fields_dimensions), 'order_by': ', '.join([ self._get_dimension_field(f)['order_by'] for f in self._get_dimension_fields(facet) ]), 'filters': self._convert_filters_to_where(filters), 'limit': LIMIT } elif facet['type'] == 'function': # 1 dim only now sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s''' % { 'database': database, 'table': table, 'fields': self._get_aggregate_function(facet), 'filters': self._convert_filters_to_where(filters), } else: fields = Collection2.get_field_list(dashboard) sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % { 'database': database, 'table': table, 'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields]) } if filters: sql += ' ' + self._convert_filters_to_where(filters) sql += ' LIMIT %s' % LIMIT editor = make_notebook(name='Execute and watch', editor_type=dashboard['engine'], statement=sql, database=database, status='ready-execute', skip_historify=True) response = editor.execute(MockRequest(self.user)) if 'handle' in response and response['handle'].get('sync'): response['result'] = self._convert_result(response['result'], dashboard, facet, query) return response