def browse(request, name, is_mobile=False): engine = request.GET.get('engine', 'solr') collections = get_engine(request.user, engine).datasets() if not collections and engine == 'solr': return no_collections(request) collection = Collection2(user=request.user, name=name, engine=engine) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} template = 'search.mako' if is_mobile: template = 'search_m.mako' return render( template, request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'autoLoad': True, 'collections': collections, 'layout': [{ "size": 12, "rows": [{ "widgets": [{ "size": 12, "name": "Grid Results", "id": "52f07188-f30f-1296-2450-f77e02e1a5c0", "widgetType": "resultset-widget", "properties": {}, "offset": 0, "isLoading": True, "klass": "card card-widget span12" }] }], "drops": ["temp"], "klass": "card card-home card-column span10" }], 'is_latest': LATEST.get(), 'engines': get_engines(request.user) }), 'is_owner': True, 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_index': can_edit_index(request.user), 'mobile': is_mobile })
def new_search(request): engine = request.GET.get('engine', 'solr') cluster = request.POST.get('cluster', '""') collections = get_engine( request.user, engine, cluster=cluster).datasets() if engine != 'report' else ['default'] if not collections: if engine == 'solr': return no_collections(request) else: return importer(request) collection = Collection2(user=request.user, name=collections[0], engine=engine) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} layout = DEFAULT_LAYOUT if engine != 'report' else REPORT_LAYOUT if request.GET.get('format', 'plain') == 'json': return JsonResponse({ 'collection': collection.get_props(request.user), 'query': query, 'initial': { 'collections': collections, 'layout': layout, 'qb_layout': QUERY_BUILDER_LAYOUT, 'text_search_layout': TEXT_SEARCH_LAYOUT, 'is_latest': _get_latest(), 'engines': get_engines(request.user) } }) else: return render( 'search.mako', request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'collections': collections, 'layout': layout, 'qb_layout': QUERY_BUILDER_LAYOUT, 'text_search_layout': TEXT_SEARCH_LAYOUT, 'is_latest': _get_latest(), 'engines': get_engines(request.user) }), 'is_owner': True, 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_index': can_edit_index(request.user), 'is_report': engine == 'report' })
def index_fields_dynamic(request): result = {'status': -1, 'message': 'Error'} try: name = request.POST['name'] engine = request.POST['engine'] dynamic_fields = get_engine(request.user, engine).luke(name) result['message'] = '' result['fields'] = [ Collection2._make_field(name, properties) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['gridlayout_header_fields'] = [ Collection2._make_gridlayout_header_field({'name': name, 'type': properties.get('type')}, True) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['status'] = 0 except Exception, e: result['message'] = force_unicode(e)
def get_collection(request): result = {'status': -1, 'message': ''} try: name = request.POST['name'] engine = request.POST['engine'] collection = Collection2(request.user, name=name, engine=engine) collection_json = collection.get_json(request.user) result['collection'] = json.loads(collection_json) result['status'] = 0 except Exception, e: result['message'] = force_unicode(e)
def index(request, is_mobile=False, is_embeddable=False): hue_collections = DashboardController(request.user).get_search_collections() collection_id = request.GET.get('collection') if not hue_collections or not collection_id: return admin_collections(request, True, is_mobile) try: collection_doc = Document2.objects.get(id=collection_id) if USE_NEW_EDITOR.get(): collection_doc.can_read_or_exception(request.user) else: collection_doc.doc.get().can_read_or_exception(request.user) collection = Collection2(request.user, document=collection_doc) except Exception, e: raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def get_fields(self, collection_or_core_name): api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) try: field_data = api.fields(collection_or_core_name) fields = self._format_flags(field_data['schema']['fields']) except Exception, e: LOG.warn('/luke call did not succeed: %s' % e) try: fields = api.schema_fields(collection_or_core_name) fields = Collection2._make_luke_from_schema_fields(fields) except: LOG.exception( _('Could not fetch fields for collection %s.') % collection_or_core_name) raise PopupException( _('Could not fetch fields for collection %s. See logs for more info.' ) % collection_or_core_name)
def get_collection(request): result = {'status': -1, 'message': ''} try: name = request.POST.get('name') engine = request.POST.get('engine') source = request.POST.get('source') collection = Collection2(request.user, name=name, engine=engine, source=source) collection_json = collection.get_json(request.user) result['collection'] = json.loads(collection_json) result['status'] = 0 except Exception as e: result['message'] = force_unicode(e) return JsonResponse(result)
def new_search(request): engine = request.GET.get('engine', 'solr') collections = get_engine(request.user, engine).datasets() if not collections: return no_collections(request) collection = Collection2(user=request.user, name=collections[0], engine=engine) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} if request.GET.get('format', 'plain') == 'json': return JsonResponse({ 'collection': collection.get_props(request.user), 'query': query, 'initial': { 'collections': collections, 'layout': DEFAULT_LAYOUT, 'is_latest': LATEST.get(), 'engines': get_engines(request.user) } }) else: return render( 'search.mako', request, { 'collection': collection, 'query': query, 'initial': json.dumps({ 'collections': collections, 'layout': DEFAULT_LAYOUT, 'is_latest': LATEST.get(), 'engines': get_engines(request.user) }), 'is_owner': True, 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_index': can_edit_index(request.user) })
def copy_collections(self, collection_ids): result = {'status': -1, 'message': ''} try: for doc2 in self.get_shared_search_collections(): if doc2.id in collection_ids: doc2 = Document2.objects.get_by_uuid(user=self.user, uuid=doc2.uuid) doc = doc2.doc.get() name = doc2.name + '-copy' doc2 = doc2.copy(name=name, owner=self.user) doc.copy(content_object=doc2, name=name, owner=self.user) collection = Collection2(self.user, document=doc2) collection.data['collection']['label'] = name doc2.update_data({'collection': collection.data['collection']}) doc2.save() result['status'] = 0 except Exception, e: LOG.exception('Error copying collection') result['message'] = unicode(str(e), "utf8")
def query(self, dashboard, query, facet=None): if query['qs'] == [{'q': '_root_:*'}]: return {'response': {'numFound': 0}} filters = [q['q'] for q in query['qs'] if q['q']] filters.extend(self._get_fq(dashboard, query, facet)) result_properties = {} timeFilter = self._get_time_filter_query(dashboard, query) if timeFilter: filters.append(timeFilter) if self.source == 'query': sql_from = '(%(query)s) t' % { 'query': self._get_query(dashboard['name']) } database, table = '', '' else: database, table = self._get_database_table_names(dashboard['name']) sql_from = '%(backticks)s%(database)s%(backticks)s.%(backticks)s%(table)s%(backticks)s' % { 'database': database, 'table': table, 'backticks': self.backticks } if facet and facet['properties']['facets']: for i, _facet in enumerate(facet['properties']['facets']): _facet['position'] = i if facet['type'] == 'nested': fields_dimensions = [ self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet) ] last_dimension_seen = False fields = [] for f in reversed(facet['properties']['facets']): if f['aggregate']['function'] == 'count': if not last_dimension_seen: fields.insert( 0, 'COUNT(*) AS %(field)s_%(position)s' % f) last_dimension_seen = True fields.insert( 0, self._get_dimension_field(f) ['name' if last_dimension_seen else 'select']) else: if not last_dimension_seen: fields.insert( 0, self._get_aggregate_function(f) + 'AS %(field)s_%(position)s' % f) has_facet_mincount_greater_than_one = [ f for f in facet['properties']['facets'] if f['mincount'] > 1 ] if has_facet_mincount_greater_than_one and self._supports_count_over( ): mincount_fields_name = [] mincount_fields_operation = [] mincount_where = [] for f in facet['properties']['facets']: mincount_fields_name.append(f['field']) mincount_field_name = 'count__' + '_'.join( mincount_fields_name) mincount_fields_operation.append( 'COUNT(*) OVER (PARTITION BY %s) AS %s' % (', '.join(mincount_fields_name), mincount_field_name)) mincount_where.append( '%s >= %s' % (mincount_field_name, str(f['mincount']))) sql_from = '''(SELECT * FROM (SELECT *, %(fields)s FROM %(sql_from)s) default WHERE %(where)s) default''' % { 'fields': ', '.join(mincount_fields_operation), 'sql_from': sql_from, 'where': ' AND '.join(mincount_where) } order_by = ', '.join([ self._get_dimension_field(f)['order_by'] for f in reversed(facet['properties']['facets']) if f['sort'] != 'default' ]) sql = '''SELECT %(fields)s FROM %(sql_from)s %(filters)s GROUP BY %(fields_dimensions)s %(order_by)s LIMIT %(limit)s''' % { 'sql_from': sql_from, 'fields': ', '.join(fields), 'fields_dimensions': ', '.join(fields_dimensions), 'order_by': 'ORDER BY %s' % order_by if order_by else '', 'filters': self._convert_filters_to_where(filters), 'limit': LIMIT } elif facet['type'] == 'function': # 1 dim only now aggregate_function = facet['properties']['facets'][0][ 'aggregate']['function'] if (aggregate_function == 'percentile' or aggregate_function == 'median') and not self._supports_percentile( ) and self._supports_cume_dist(): sql_from = ''' (SELECT * FROM ( SELECT %(field)s, cume_dist() OVER (ORDER BY %(field)s) * 100 AS cume_dist__%(field)s FROM %(sql_from)s ) DEFAULT WHERE cume_dist__%(field)s >= %(value)s) DEFAULT ''' % { 'field': facet['properties']['facets'][0]['field'], 'value': facet['properties']['facets'][0]['aggregate'] ['percentile'] if aggregate_function == 'percentile' else 50, 'sql_from': sql_from, } sql = '''SELECT %(fields)s FROM %(sql_from)s %(filters)s''' % { 'sql_from': sql_from, 'fields': self._get_aggregate_function( facet['properties']['facets'][0]), 'filters': self._convert_filters_to_where(filters), } elif facet['type'] == 'statement': doc = Document2.objects.get_by_uuid( user=self.user, uuid=facet['properties']['statementUuid'], perm_type='read') snippets = doc.data_dict.get('snippets', []) sql = snippets[0].get('statement', '') result_properties = facet['properties']['result'] else: fields = Collection2.get_field_list(dashboard) order_by = ', '.join([ '%(backticks)s%(name)s%(backticks)s %(direction)s' % { 'backticks': self.backticks, 'name': f['name'], 'direction': f['sort']['direction'] } for f in dashboard['template']['fieldsAttributes'] if f['sort']['direction'] and f['name'] in fields ]) sql = ''' SELECT %(fields)s FROM %(sql_from)s %(filters)s %(order_by)s %(limit)s''' % { 'sql_from': sql_from, 'fields': ', '.join([ '%(backticks)s%(column)s%(backticks)s as %(backticks)s%(column)s%(backticks)s' % { 'backticks': self.backticks, 'column': f } if f != '*' else '*' for f in fields ]), 'filters': self._convert_filters_to_where(filters) if filters else '', 'order_by': 'ORDER BY %s' % order_by if order_by else '', 'limit': 'LIMIT %s' % dashboard['template']['rows'] or LIMIT } editor = make_notebook(name='Execute and watch', editor_type=dashboard['engine'], statement=sql, database=database, status='ready-execute', skip_historify=True, result_properties=result_properties) response = editor.execute(MockRequest(self.user, self.cluster)) if 'handle' in response and response['handle'].get('sync'): response['result'] = self._convert_result(response['result'], dashboard, facet, query) return response
def luke(self, collection): fields = self.schema_fields(collection) return {'fields': Collection2._make_luke_from_schema_fields(fields)}
def query(self, dashboard, query, facet=None): database, table = self._get_database_table_names(dashboard['name']) if query['qs'] == [{'q': '_root_:*'}]: return {'response': {'numFound': 0}} filters = [q['q'] for q in query['qs'] if q['q']] filters.extend(self._get_fq(dashboard, query, facet)) timeFilter = self._get_time_filter_query(dashboard, query) if timeFilter: filters.append(timeFilter) if facet: if facet['type'] == 'nested': fields_dimensions = [self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet)] last_dimension_seen = False fields = [] for f in reversed(facet['properties']['facets']): if f['aggregate']['function'] == 'count': if not last_dimension_seen: fields.insert(0, 'COUNT(*) AS Count') last_dimension_seen = True fields.insert(0, self._get_dimension_field(f)['select']) else: if not last_dimension_seen: fields.insert(0, self._get_aggregate_function(f)) if not last_dimension_seen: fields.insert(0, 'COUNT(*) as Count') fields.insert(0, self._get_dimension_field(facet)['select']) sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s GROUP BY %(fields_dimensions)s ORDER BY %(order_by)s LIMIT %(limit)s''' % { 'database': database, 'table': table, 'fields': ', '.join(fields), 'fields_dimensions': ', '.join(fields_dimensions), 'order_by': ', '.join([self._get_dimension_field(f)['order_by'] for f in self._get_dimension_fields(facet)]), 'filters': self._convert_filters_to_where(filters), 'limit': LIMIT } elif facet['type'] == 'function': # 1 dim only now sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s''' % { 'database': database, 'table': table, 'fields': self._get_aggregate_function(facet), 'filters': self._convert_filters_to_where(filters), } else: fields = Collection2.get_field_list(dashboard) sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % { 'database': database, 'table': table, 'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields]) } if filters: sql += ' ' + self._convert_filters_to_where(filters) sql += ' LIMIT %s' % LIMIT editor = make_notebook( name='Execute and watch', editor_type=dashboard['engine'], statement=sql, database=database, status='ready-execute', skip_historify=True ) response = editor.execute(MockRequest(self.user)) if 'handle' in response and response['handle'].get('sync'): response['result'] = self._convert_result(response['result'], dashboard, facet, query) return response
def index(request, is_mobile=False): engine = request.GET.get('engine', 'solr') cluster = request.POST.get('cluster', '""') collection_id = request.GET.get('collection') collections = get_engine( request.user, engine, cluster=cluster).datasets() if engine != 'report' else ['default'] if not collections: if engine == 'solr': return no_collections(request) else: return importer(request) try: collection_doc = Document2.objects.get(id=collection_id) if USE_NEW_EDITOR.get(): collection_doc.can_read_or_exception(request.user) else: collection_doc.doc.get().can_read_or_exception(request.user) collection = Collection2(request.user, document=collection_doc) except Exception as e: raise PopupException( e, title= _("Dashboard does not exist or you don't have the permission to access it." )) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} if request.method == 'GET': if 'q' in request.GET: query['qs'][0]['q'] = antixss(request.GET.get('q', '')) if 'qd' in request.GET: query['qd'] = antixss(request.GET.get('qd', '')) template = 'search.mako' if is_mobile: template = 'search_m.mako' engine = collection.data['collection'].get('engine', 'solr') return render( template, request, { 'collection': collection, 'query': json.dumps(query), 'initial': json.dumps({ 'collections': collections, 'layout': DEFAULT_LAYOUT, 'qb_layout': QUERY_BUILDER_LAYOUT, 'text_search_layout': TEXT_SEARCH_LAYOUT, 'is_latest': _get_latest(), 'engines': get_engines(request.user) }), 'is_owner': collection_doc.can_write(request.user) if USE_NEW_EDITOR.get() else collection_doc.doc.get().can_write(request.user), 'can_edit_index': can_edit_index(request.user), 'is_embeddable': request.GET.get('is_embeddable', False), 'mobile': is_mobile, 'is_report': collection.data['collection'].get('engine') == 'report' })
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type, window_size): properties = { 'sort': 'desc', 'canRange': False, 'stacked': False, 'limit': 10, 'mincount': 0, 'missing': False, 'isDate': False, 'slot': 0, 'aggregate': { 'function': 'unique', 'formula': '', 'plain_formula': '', 'percentile': 50 } } template = { "showFieldList": True, "showGrid": False, "showChart": True, "chartSettings": { 'chartType': 'pie' if widget_type == 'pie2-widget' else ('timeline' if widget_type == 'timeline-widget' else ('gradientmap' if widget_type == 'gradient-map-widget' else 'bars' )), 'chartSorting': 'none', 'chartScatterGroup': None, 'chartScatterSize': None, 'chartScope': 'world', 'chartX': None, 'chartYSingle': None, 'chartYMulti': [], 'chartData': [], 'chartMapLabel': None, 'chartSelectorType': 'bar' }, "fieldsAttributes": [], "fieldsAttributesFilter": "", "filteredAttributeFieldsAll": True, "fields": [], "fieldsSelected": [], "leafletmap": { 'latitudeField': None, 'longitudeField': None, 'labelField': None }, # Use own? 'leafletmapOn': False, 'isGridLayout': False, "hasDataForChart": True, "rows": 25, } if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): facet_type = 'pivot' elif widget_type == 'document-widget': # SQL query, 1 solr widget if collection['selectedDocument'].get('uuid'): doc = Document2.objects.get_by_uuid( user=user, uuid=collection['selectedDocument']['uuid'], perm_type='read') snippets = doc.data_dict.get('snippets', []) if snippets: table_metadata = get_api(MockRequest(user, '""'), snippets[0]).autocomplete( { 'source': 'query', 'type': snippets[0]['type'] }, doc.id) template['fieldsAttributes'] = [ Collection2._make_gridlayout_header_field(field) for field in table_metadata['extended_columns'] ] properties['engine'] = snippets[0]['type'] if snippets[0]['result']['handle']: handle = snippets[0]['result']['handle'] # Replace previous_statement_hash so that we rerun current statement properties['result'] = { 'handle': { 'statement_id': handle['statement_id'], 'statements_count': handle['statements_count'], 'previous_statement_hash': hashlib.sha224(str(uuid.uuid4())).hexdigest() } } else: properties['result'] = {'handle': {}} properties['statement'] = snippets[0]['statement_raw'] else: properties['result'] = {'handle': {}} properties['statement'] = '' else: # Demo data for now properties['statement'] = 'select * from customers' properties['result'] = {'handle': {}} properties['uuid'] = facet_field properties['facets'] = [{ 'canRange': False, 'field': 'blank', 'limit': 10, 'mincount': 0, 'sort': 'desc', 'aggregate': { 'function': 'count' }, 'isDate': False, 'type': 'field' }] facet_type = 'statement' else: api = get_engine(user, collection) range_properties = _new_range_facet(api, collection, facet_field, widget_type, window_size) if range_properties: facet_type = 'range' properties.update(range_properties) properties['initial_gap'] = properties['gap'] properties['initial_start'] = properties['start'] properties['initial_end'] = properties['end'] else: facet_type = 'field' if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget', 'tree2-widget', 'text-facet-widget', 'hit-widget', 'gradient-map-widget'): # properties = {'canRange': False, 'stacked': False, 'limit': 10} # TODO: Lighter weight top nested facet properties['facets_form'] = NESTED_FACET_FORM # Not supported on dim 2 currently properties['facets_form']['type'] = 'field' properties['facets_form']['canRange'] = False properties['facets_form']['isFacetForm'] = True facet = NESTED_FACET_FORM.copy() facet['field'] = facet_field facet['limit'] = 10 facet['fieldLabel'] = facet_field facet['multiselect'] = True if widget_type == 'text-facet-widget': properties['type'] = facet_type if USE_GRIDSTER.get(): properties['limit'] = facet['limit'] = 100 if range_properties: # TODO: timeline still uses properties from top properties facet.update(range_properties) facet['initial_gap'] = facet['gap'] facet['initial_start'] = facet['start'] facet['initial_end'] = facet['end'] facet['stacked'] = False facet['type'] = 'range' else: facet['type'] = facet_type if collection.get('engine', 'solr') != 'solr': facet['sort'] = 'default' properties['facets'] = [facet] properties['domain'] = {'blockParent': [], 'blockChildren': []} properties['compare'] = COMPARE_FACET properties['filter'] = QUERY_FACET if widget_type == 'hit-widget': facet_type = 'function' facet['aggregate']['function'] = 'unique' else: facet_type = 'nested' facet['aggregate']['function'] = 'count' if widget_type == 'pie2-widget': properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' elif widget_type == 'tree2-widget': properties['scope'] = 'tree' properties['facets_form']['limit'] = 5 properties['isOldPivot'] = True elif widget_type == 'gradient-map-widget': properties['scope'] = 'world' facet['limit'] = 100 else: properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget') and widget_type != 'gradient-map-widget': properties['mincount'] = 1 properties['facets'] = [] properties['stacked'] = True properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5} if widget_type == 'map-widget': properties['scope'] = 'world' properties['limit'] = 100 else: properties[ 'scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree' if widget_type == 'histogram-widget': properties['enableSelection'] = True properties['timelineChartType'] = 'bar' properties['extraSeries'] = [] return { 'id': facet_id, 'label': facet_label, 'field': facet_field, 'type': facet_type, 'widgetType': widget_type, 'properties': properties, # Hue 4+ 'template': template, 'queryResult': {} }
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) params = self._get_params() + ( ('q', self._get_q(query)), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) json_facets = {} timeFilter = self._get_range_borders(collection, query) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']), ) elif facet['type'] == 'range' or facet['type'] == 'range-up': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], 'mincount': int(facet['properties']['mincount']) } if timeFilter and timeFilter['time_field'] == facet[ 'field'] and ( facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'): keys.update( self._get_time_filter_query(timeFilter, facet)) params += (( 'facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'field': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0), 'mincount': int(facet['properties']['mincount']) } params += (( 'facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'nested': _f = { 'field': facet['field'], 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'text-facet-widget' else 0), 'mincount': int(facet['properties']['mincount']), 'sort': { 'count': facet['properties']['sort'] }, } if facet['properties']['domain'].get( 'blockParent' ) or facet['properties']['domain'].get('blockChildren'): _f['domain'] = {} if facet['properties']['domain'].get('blockParent'): _f['domain']['blockParent'] = ' OR '.join( facet['properties']['domain']['blockParent']) if facet['properties']['domain'].get('blockChildren'): _f['domain']['blockChildren'] = ' OR '.join( facet['properties']['domain']['blockChildren']) if 'start' in facet['properties'] and not facet[ 'properties'].get('type') == 'field': _f.update({ 'type': 'range', 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], }) if timeFilter and timeFilter['time_field'] == facet[ 'field'] and ( facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'bucket-widget'): _f.update( self._get_time_filter_query(timeFilter, facet)) else: _f.update({ 'type': 'terms', 'field': facet['field'], 'excludeTags': facet['id'], 'offset': 0, 'numBuckets': True, 'allBuckets': True, #'prefix': '' # Forbidden on numeric fields }) if facet['properties']['canRange'] and not facet[ 'properties']['isDate']: del _f['mincount'] # Numeric fields do not support if facet['properties']['facets']: self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1) if facet['widgetType'] == 'text-facet-widget': _fname = _f['facet'].keys()[0] _f['sort'] = {_fname: facet['properties']['sort']} # domain = '-d2:NaN' # Solr 6.4 json_facets[facet['id']] = _f elif facet['type'] == 'function': json_facets[facet['id']] = self._get_aggregate_function( facet) json_facets['processEmpty'] = True elif facet['type'] == 'pivot': if facet['properties']['facets'] or facet[ 'widgetType'] == 'map-widget': fields = facet['field'] fields_limits = [] for f in facet['properties']['facets']: fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit'])) fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount'])) fields += ',' + f['field'] keys = { 'id': '%(id)s' % facet, 'key': '%(field)s-%(id)s' % facet, 'field': facet['field'], 'fields': fields, 'limit': int(facet['properties'].get('limit', 10)), 'mincount': int(facet['properties']['mincount']), 'fields_limits': ' '.join(fields_limits) } params += (( 'facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys), ) if json_facets: params += (('json.facet', json.dumps(json_facets)), ) params += self._get_fq(collection, query) from dashboard.models import Collection2 fl = urllib.unquote( utf_quoter(','.join(Collection2.get_field_list(collection)))) nested_fields = self._get_nested_fields(collection) if nested_fields: fl += urllib.unquote( utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields))) params += (('fl', fl), ) params += ( ('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 5), ('hl.fragsize', 1000), ) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter( lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append( '%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += (('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) params = self._get_params() + ( ('q', self._get_q(query)), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) json_facets = {} timeFilter = self._get_range_borders(collection, query) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']),) elif facet['type'] == 'range' or facet['type'] == 'range-up': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], 'mincount': int(facet['properties']['mincount']) } if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'): keys.update(self._get_time_filter_query(timeFilter, facet)) params += ( ('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'field': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0), 'mincount': int(facet['properties']['mincount']) } params += ( ('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'nested': _f = {} if facet['properties']['facets']: self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1, timeFilter) if facet['properties'].get('domain'): if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'): _f['domain'] = {} if facet['properties']['domain'].get('blockParent'): _f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent']) if facet['properties']['domain'].get('blockChildren'): _f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren']) if _f: sort = {'count': facet['properties']['facets'][0]['sort']} for i, agg in enumerate(self._get_dimension_aggregates(facet['properties']['facets'][1:])): if agg['sort'] != 'default': agg_function = self._get_aggregate_function(agg) sort = {'agg_%02d_%02d:%s' % (1, i, agg_function): agg['sort']} if sort.get('count') == 'default': sort['count'] = 'desc' dim_key = [key for key in _f['facet'].keys() if 'dim' in key][0] _f['facet'][dim_key].update({ 'excludeTags': facet['id'], 'offset': 0, 'numBuckets': True, 'allBuckets': True, 'sort': sort #'prefix': '' # Forbidden on numeric fields }) json_facets[facet['id']] = _f['facet'][dim_key] elif facet['type'] == 'function': if facet['properties']['facets']: json_facets[facet['id']] = self._get_aggregate_function(facet['properties']['facets'][0]) if facet['properties']['compare']['is_enabled']: # TODO: global compare override unit = re.split('\d+', facet['properties']['compare']['gap'])[1] json_facets[facet['id']] = { 'type': 'range', 'field': collection['timeFilter'].get('field'), 'start': 'NOW/%s-%s-%s' % (unit, facet['properties']['compare']['gap'], facet['properties']['compare']['gap']), 'end': 'NOW/%s' % unit, 'gap': '+%(gap)s' % facet['properties']['compare'], 'facet': {facet['id']: json_facets[facet['id']]} } if facet['properties']['filter']['is_enabled']: json_facets[facet['id']] = { 'type': 'query', 'q': facet['properties']['filter']['query'] or EMPTY_QUERY.get(), 'facet': {facet['id']: json_facets[facet['id']]} } json_facets['processEmpty'] = True elif facet['type'] == 'pivot': if facet['properties']['facets'] or facet['widgetType'] == 'map-widget': fields = facet['field'] fields_limits = [] for f in facet['properties']['facets']: fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit'])) fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount'])) fields += ',' + f['field'] keys = { 'id': '%(id)s' % facet, 'key': '%(field)s-%(id)s' % facet, 'field': facet['field'], 'fields': fields, 'limit': int(facet['properties'].get('limit', 10)), 'mincount': int(facet['properties']['mincount']), 'fields_limits': ' '.join(fields_limits) } params += ( ('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys), ) if json_facets: params += ( ('json.facet', json.dumps(json_facets)), ) params += self._get_fq(collection, query) fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection)))) nested_fields = self._get_nested_fields(collection) if nested_fields: fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields))) if collection['template']['moreLikeThis'] and fl != ['*']: # Potential conflict with nested documents id_field = collection.get('idField', 'id') params += ( ('mlt', 'true'), ('mlt.fl', fl.replace(',%s' % id_field, '')), ('mlt.mintf', 1), ('mlt.mindf', 1), ('mlt.maxdf', 50), ('mlt.maxntp', 1000), ('mlt.count', 10), #('mlt.minwl', 1), #('mlt.maxwl', 1), ) fl = '*' params += (('fl', fl),) params += ( ('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 5), ('hl.fragsize', 1000), ) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append('%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += ( ('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def query(self, dashboard, query, facet=None): database, table = self._get_database_table_names(dashboard['name']) if query['qs'] == [{'q': '_root_:*'}]: return {'response': {'numFound': 0}} filters = [q['q'] for q in query['qs'] if q['q']] filters.extend(self._get_fq(dashboard, query, facet)) timeFilter = self._get_time_filter_query(dashboard, query) if timeFilter: filters.append(timeFilter) if facet: if facet['type'] == 'nested': fields_dimensions = [ self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet) ] last_dimension_seen = False fields = [] for f in reversed(facet['properties']['facets']): if f['aggregate']['function'] == 'count': if not last_dimension_seen: fields.insert(0, 'COUNT(*) AS Count') last_dimension_seen = True fields.insert(0, self._get_dimension_field(f)['select']) else: if not last_dimension_seen: fields.insert(0, self._get_aggregate_function(f)) if not last_dimension_seen: fields.insert(0, 'COUNT(*) as Count') fields.insert(0, self._get_dimension_field(facet)['select']) sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s GROUP BY %(fields_dimensions)s ORDER BY %(order_by)s LIMIT %(limit)s''' % { 'database': database, 'table': table, 'fields': ', '.join(fields), 'fields_dimensions': ', '.join(fields_dimensions), 'order_by': ', '.join([ self._get_dimension_field(f)['order_by'] for f in self._get_dimension_fields(facet) ]), 'filters': self._convert_filters_to_where(filters), 'limit': LIMIT } elif facet['type'] == 'function': # 1 dim only now sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s''' % { 'database': database, 'table': table, 'fields': self._get_aggregate_function(facet), 'filters': self._convert_filters_to_where(filters), } else: fields = Collection2.get_field_list(dashboard) sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % { 'database': database, 'table': table, 'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields]) } if filters: sql += ' ' + self._convert_filters_to_where(filters) sql += ' LIMIT %s' % LIMIT editor = make_notebook(name='Execute and watch', editor_type=dashboard['engine'], statement=sql, database=database, status='ready-execute', skip_historify=True) response = editor.execute(MockRequest(self.user)) if 'handle' in response and response['handle'].get('sync'): response['result'] = self._convert_result(response['result'], dashboard, facet, query) return response
def copy_document(request): uuid = json.loads(request.POST.get('uuid'), '""') if not uuid: raise PopupException(_('copy_document requires uuid')) # Document2 and Document model objects are linked and both are saved when saving document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid) # Document model object document1 = document.doc.get() if document.type == 'directory': raise PopupException(_('Directory copy is not supported')) name = document.name + '-copy' # Make the copy of the Document2 model object copy_document = document.copy(name=name, owner=request.user) # Make the copy of Document model object too document1.copy(content_object=copy_document, name=name, owner=request.user) # Import workspace for all oozie jobs if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2': from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace # Update the name field in the json 'data' field if document.type == 'oozie-workflow2': workflow = Workflow(document=document) workflow.update_name(name) workflow.update_uuid(copy_document.uuid) _import_workspace(request.fs, request.user, workflow) copy_document.update_data( {'workflow': workflow.get_data()['workflow']}) copy_document.save() if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2': if document.type == 'oozie-bundle2': bundle_or_coordinator = Bundle(document=document) else: bundle_or_coordinator = Coordinator(document=document) json_data = bundle_or_coordinator.get_data_for_json() json_data['name'] = name json_data['uuid'] = copy_document.uuid copy_document.update_data(json_data) copy_document.save() _import_workspace(request.fs, request.user, bundle_or_coordinator) elif document.type == 'search-dashboard': from dashboard.models import Collection2 collection = Collection2(request.user, document=document) collection.data['collection']['label'] = name collection.data['collection']['uuid'] = copy_document.uuid copy_document.update_data( {'collection': collection.data['collection']}) copy_document.save() # Keep the document and data in sync else: copy_data = copy_document.data_dict if 'name' in copy_data: copy_data['name'] = name if 'uuid' in copy_data: copy_data['uuid'] = copy_document.uuid copy_document.update_data(copy_data) copy_document.save() return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def query(self, dashboard, query, facet=None): database, table = self._get_database_table_names(dashboard['name']) if query['qs'] == [{'q': '_root_:*'}]: return {'response': {'numFound': 0}} filters = [q['q'] for q in query['qs'] if q['q']] filters.extend(self._get_fq(dashboard, query, facet)) timeFilter = self._get_time_filter_query(dashboard, query) if timeFilter: filters.append(timeFilter) if facet and facet['properties']['facets']: for i, _facet in enumerate(facet['properties']['facets']): _facet['position'] = i if facet['type'] == 'nested': fields_dimensions = [ self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet) ] last_dimension_seen = False fields = [] for f in reversed(facet['properties']['facets']): if f['aggregate']['function'] == 'count': if not last_dimension_seen: fields.insert( 0, 'COUNT(*) AS %(field)s_%(position)s' % f) last_dimension_seen = True fields.insert( 0, self._get_dimension_field(f) ['name' if last_dimension_seen else 'select']) else: if not last_dimension_seen: fields.insert( 0, self._get_aggregate_function(f) + 'AS %(field)s_%(position)s' % f) if self._supports_count_over(): mincount_fields_name = [] mincount_fields_operation = [] mincount_where = [] for f in facet['properties']['facets']: mincount_fields_name.append(f['field']) mincount_field_name = '_'.join( mincount_fields_name) + "_count" mincount_fields_operation.append( 'count(*) over (partition by %s) as %s' % (','.join(mincount_fields_name), mincount_field_name)) mincount_where.append( '%s >= %s' % (mincount_field_name, str(f['mincount']))) sql_mincount = '''(SELECT * FROM (SELECT *,%(fields)s FROM %(database)s.%(table)s) default WHERE %(where)s) default''' % { 'fields': ', '.join(mincount_fields_operation), 'database': database, 'table': table, 'where': ' and '.join(mincount_where) } else: sql_mincount = '(%(database)s.%(table)s)' % { 'database': database, 'table': table } order_by = ', '.join([ self._get_dimension_field(f)['order_by'] for f in reversed(facet['properties']['facets']) if f['sort'] != 'default' ]) sql = '''SELECT %(fields)s FROM %(sql_mincount)s %(filters)s GROUP BY %(fields_dimensions)s %(order_by)s LIMIT %(limit)s''' % { 'sql_mincount': sql_mincount, 'database': database, 'table': table, 'fields': ', '.join(fields), 'fields_dimensions': ', '.join(fields_dimensions), 'order_by': 'ORDER BY %s' % order_by if order_by else '', 'filters': self._convert_filters_to_where(filters), 'limit': LIMIT } elif facet['type'] == 'function': # 1 dim only now sql = '''SELECT %(fields)s FROM %(database)s.%(table)s %(filters)s''' % { 'database': database, 'table': table, 'fields': self._get_aggregate_function( facet['properties']['facets'][0]), 'filters': self._convert_filters_to_where(filters), } else: fields = Collection2.get_field_list(dashboard) sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % { 'database': database, 'table': table, 'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields]) } if filters: sql += ' ' + self._convert_filters_to_where(filters) sql += ' LIMIT %s' % dashboard['template']['rows'] or LIMIT editor = make_notebook(name='Execute and watch', editor_type=dashboard['engine'], statement=sql, database=database, status='ready-execute', skip_historify=True) response = editor.execute(MockRequest(self.user)) if 'handle' in response and response['handle'].get('sync'): response['result'] = self._convert_result(response['result'], dashboard, facet, query) return response