def index_fields_dynamic(request):
result = {'status': -1, 'message': 'Error'}
try:
name = request.POST['name']
dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name)
result['message'] = ''
result['fields'] = [
Collection2._make_field(name, properties)
for name, properties in dynamic_fields['fields'].iteritems()
if 'dynamicBase' in properties
]
result['gridlayout_header_fields'] = [
Collection2._make_gridlayout_header_field(
{
'name': name,
'type': properties.get('type')
}, True)
for name, properties in dynamic_fields['fields'].iteritems()
if 'dynamicBase' in properties
]
result['status'] = 0
except Exception, e:
result['message'] = force_unicode(e)
def browse(request, name, is_mobile=False):
collections = SearchController(request.user).get_all_indexes()
if not collections:
return no_collections(request)
collection = Collection2(user=request.user, name=name)
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
template = 'search.mako'
if is_mobile:
template = 'search_m.mako'
return render(template, request, {
'collection': collection,
'query': query,
'initial': json.dumps({
'autoLoad': True,
'collections': collections,
'layout': [
{"size":12,"rows":[{"widgets":[
{"size":12,"name":"Grid Results","id":"52f07188-f30f-1296-2450-f77e02e1a5c0","widgetType":"resultset-widget",
"properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
"drops":["temp"],"klass":"card card-home card-column span10"}
],
'is_latest': LATEST.get()
}),
'is_owner': True,
'can_edit_index': can_edit_index(request.user),
'mobile': is_mobile
})
def new_search(request, is_embeddable=False):
collections = SearchController(request.user).get_all_indexes()
if not collections:
return no_collections(request)
collection = Collection2(user=request.user, name=collections[0])
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
template = 'search.mako'
if is_embeddable:
template = 'search_embeddable.mako'
return render(template, request, {
'collection': collection,
'query': query,
'initial': json.dumps({
'collections': collections,
'layout': [
{"size":2,"rows":[{"widgets":[]}],"drops":["temp"],"klass":"card card-home card-column span2"},
{"size":10,"rows":[{"widgets":[
{"size":12,"name":"Filter Bar","widgetType":"filter-widget", "id":"99923aef-b233-9420-96c6-15d48293532b",
"properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]},
{"widgets":[
{"size":12,"name":"Grid Results","widgetType":"resultset-widget", "id":"14023aef-b233-9420-96c6-15d48293532b",
"properties":{},"offset":0,"isLoading":True,"klass":"card card-widget span12"}]}],
"drops":["temp"],"klass":"card card-home card-column span10"},
],
'is_latest': LATEST.get(),
}),
'is_owner': True,
'can_edit_index': can_edit_index(request.user)
})
def copy_collections(self, collection_ids):
result = {'status': -1, 'message': ''}
try:
for doc2 in self.get_shared_search_collections():
if doc2.id in collection_ids:
name = doc2.name + '-copy'
doc2.pk = None
doc2.id = None
doc2.uuid = str(uuid.uuid4())
doc2.name = name
doc2.owner = self.user
doc2.save()
copy_doc = Document.objects.link(doc2,
owner=copy.owner,
name=copy.name,
description=copy.description)
copy = Collection2(self.user, document=doc2)
copy.data['collection']['label'] = name
doc2.update_data({'collection': copy.data['collection']})
doc2.save()
result['status'] = 0
except Exception, e:
LOG.exception('Error copying collection')
result['message'] = unicode(str(e), "utf8")
def copy_collections(self, collection_ids):
result = {'status': -1, 'message': ''}
try:
for doc2 in self.get_shared_search_collections():
if doc2.id in collection_ids:
name = doc2.name + '-copy'
copy_doc = doc2.doc.get().copy(name=name, owner=self.user)
doc2.pk = None
doc2.id = None
doc2.uuid = str(uuid.uuid4())
doc2.name = name
doc2.owner = self.user
doc2.save()
doc2.doc.all().delete()
doc2.doc.add(copy_doc)
doc2.save()
copy = Collection2(self.user, document=doc2)
copy.data['collection']['label'] = name
doc2.update_data({'collection': copy.data['collection']})
doc2.save()
result['status'] = 0
except Exception, e:
print e
LOG.warn('Error copying collection: %s' % e)
result['message'] = unicode(str(e), "utf8")
def new_search(request):
collections = SearchController(request.user).get_all_indexes()
if not collections:
return no_collections(request)
collection = Collection2(user=request.user, name=collections[0])
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
return render(
'search.mako', request, {
'collection':
collection,
'query':
query,
'initial':
json.dumps({
'collections':
collections,
'layout': [
{
"size": 2,
"rows": [{
"widgets": []
}],
"drops": ["temp"],
"klass": "card card-home card-column span2"
},
{
"size":
10,
"rows": [{
"widgets": [{
"size": 12,
"name": "Filter Bar",
"widgetType": "filter-widget",
"properties": {},
"offset": 0,
"isLoading": True,
"klass": "card card-widget span12"
}]
}, {
"widgets": [{
"size": 12,
"name": "Grid Results",
"widgetType": "resultset-widget",
"properties": {},
"offset": 0,
"isLoading": True,
"klass": "card card-widget span12"
}]
}],
"drops": ["temp"],
"klass":
"card card-home card-column span10"
},
]
}),
'is_owner':
True
})
def index_fields_dynamic(request):
result = {'status': -1, 'message': 'Error'}
try:
name = request.POST['name']
dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name)
result['message'] = ''
result['fields'] = [Collection2._make_field(name, properties)
for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties]
result['gridlayout_header_fields'] = [Collection2._make_gridlayout_header_field({'name': name}, True)
for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties]
result['status'] = 0
except Exception, e:
result['message'] = force_unicode(e)
def copy_collections(self, collection_ids):
result = {'status': -1, 'message': ''}
try:
for collection in self.get_shared_search_collections().filter(
id__in=collection_ids):
doc2 = Document2.objects.get(type='search-dashboard',
id=collection.id)
name = doc2.name + '-copy'
copy_doc = doc2.doc.get().copy(name=name, owner=self.user)
doc2.pk = None
doc2.id = None
doc2.uuid = str(uuid.uuid4())
doc2.name = name
doc2.owner = self.user
doc2.save()
doc2.doc.all().delete()
doc2.doc.add(copy_doc)
doc2.save()
copy = Collection2(document=doc2)
copy['collection']['label'] = name
doc2.update_data({'collection': copy['collection']})
doc2.save()
result['status'] = 0
except Exception, e:
LOG.warn('Error copying collection: %s' % e)
result['message'] = unicode(str(e), "utf8")
def test_query(self):
collection = Collection2(user=self.user, name='log_analytics_demo')
collection = json.loads(collection.get_json(self.user))
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
SolrApi(SOLR_URL.get(), self.user).query(collection['collection'],
query)
def index_fields_dynamic(request):
result = {"status": -1, "message": "Error"}
try:
name = request.POST["name"]
dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name)
result["message"] = ""
result["fields"] = [
Collection2._make_field(name, properties)
for name, properties in dynamic_fields["fields"].iteritems()
if "dynamicBase" in properties
]
result["gridlayout_header_fields"] = [
Collection2._make_gridlayout_header_field({"name": name, "type": properties.get("type")}, True)
for name, properties in dynamic_fields["fields"].iteritems()
if "dynamicBase" in properties
]
result["status"] = 0
except Exception, e:
result["message"] = force_unicode(e)
def index(request):
hue_collections = SearchController(request.user).get_search_collections()
collection_id = request.GET.get('collection')
if not hue_collections or not collection_id:
return admin_collections(request, True)
try:
collection_doc = Document2.objects.get(id=collection_id)
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception, e:
raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def get_collection(request):
result = {'status': -1, 'message': ''}
try:
name = request.POST['name']
collection = Collection2(request.user, name=name)
collection_json = collection.get_json(request.user)
result['collection'] = json.loads(collection_json)
result['status'] = 0
except Exception, e:
result['message'] = force_unicode(e)
def get_fields(self, collection_or_core_name):
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
try:
field_data = api.fields(collection_or_core_name)
fields = self._format_flags(field_data['schema']['fields'])
except Exception, e:
LOG.warn('/luke call did not succeed: %s' % e)
try:
fields = api.schema_fields(collection_or_core_name)
fields = Collection2._make_luke_from_schema_fields(fields)
except:
LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name)
raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name)
def get_fields(self, collection_or_core_name):
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
try:
field_data = api.fields(collection_or_core_name)
fields = self._format_flags(field_data['schema']['fields'])
except Exception, e:
LOG.warn('/luke call did not succeed: %s' % e)
try:
fields = api.schema_fields(collection_or_core_name)
fields = Collection2._make_luke_from_schema_fields(fields)
except:
LOG.exception(
_('Could not fetch fields for collection %s.') %
collection_or_core_name)
raise PopupException(
_('Could not fetch fields for collection %s. See logs for more info.'
) % collection_or_core_name)
def copy_collections(self, collection_ids):
result = {'status': -1, 'message': ''}
try:
for doc2 in self.get_shared_search_collections():
if doc2.id in collection_ids:
doc2 = Document2.objects.get(uuid=doc2.uuid)
doc = doc2.doc.get()
name = doc2.name + '-copy'
doc2 = doc2.copy(name=name, owner=self.user)
doc.copy(content_object=doc2, name=name, owner=self.user)
collection = Collection2(self.user, document=doc2)
collection.data['collection']['label'] = name
doc2.update_data({'collection': collection.data['collection']})
doc2.save()
result['status'] = 0
except Exception, e:
LOG.exception('Error copying collection')
result['message'] = unicode(str(e), "utf8")
def luke(self, collection):
fields = self.schema_fields(collection)
return {'fields': Collection2._make_luke_from_schema_fields(fields)}
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
params = self._get_params() + (
('q', self._get_q(query)),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
json_facets = {}
timeFilter = self._get_range_borders(collection, query)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']),)
elif facet['type'] == 'range' or facet['type'] == 'range-up':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
'mincount': int(facet['properties']['mincount'])
}
if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'):
keys.update(self._get_time_filter_query(timeFilter, facet))
params += (
('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'field':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0),
'mincount': int(facet['properties']['mincount'])
}
params += (
('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'nested':
_f = {
'field': facet['field'],
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'text-facet-widget' else 0),
'mincount': int(facet['properties']['mincount']),
'sort': {'count': facet['properties']['sort']},
}
if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'):
_f['domain'] = {}
if facet['properties']['domain'].get('blockParent'):
_f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent'])
if facet['properties']['domain'].get('blockChildren'):
_f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren'])
if 'start' in facet['properties'] and not facet['properties'].get('type') == 'field':
_f.update({
'type': 'range',
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
})
if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'bucket-widget'):
_f.update(self._get_time_filter_query(timeFilter, facet))
else:
_f.update({
'type': 'terms',
'field': facet['field'],
'excludeTags': facet['id'],
'offset': 0,
'numBuckets': True,
'allBuckets': True,
#'prefix': '' # Forbidden on numeric fields
})
if facet['properties']['canRange'] and not facet['properties']['isDate']:
del _f['mincount'] # Numeric fields do not support
if facet['properties']['facets']:
self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1)
if facet['widgetType'] == 'text-facet-widget':
_fname = _f['facet'].keys()[0]
_f['sort'] = {_fname: facet['properties']['sort']}
# domain = '-d2:NaN' # Solr 6.4
json_facets[facet['id']] = _f
elif facet['type'] == 'function':
json_facets[facet['id']] = self._get_aggregate_function(facet)
json_facets['processEmpty'] = True
elif facet['type'] == 'pivot':
if facet['properties']['facets'] or facet['widgetType'] == 'map-widget':
fields = facet['field']
fields_limits = []
for f in facet['properties']['facets']:
fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit']))
fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount']))
fields += ',' + f['field']
keys = {
'id': '%(id)s' % facet,
'key': '%(field)s-%(id)s' % facet,
'field': facet['field'],
'fields': fields,
'limit': int(facet['properties'].get('limit', 10)),
'mincount': int(facet['properties']['mincount']),
'fields_limits': ' '.join(fields_limits)
}
params += (
('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys),
)
if json_facets:
params += (
('json.facet', json.dumps(json_facets)),
)
params += self._get_fq(collection, query)
from search.models import Collection2
fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection))))
nested_fields = self._get_nested_fields(collection)
if nested_fields:
fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields)))
params += (('fl', fl),)
params += (
('hl', 'true'),
('hl.fl', '*'),
('hl.snippets', 5),
('hl.fragsize', 1000),
)
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
if fields:
params += (
('sort', ','.join(fields)),
)
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def query(self, dashboard, query, facet=None):
database, table = self._get_database_table_names(dashboard['name'])
if query['qs'] == [{'q': '_root_:*'}]:
return {'response': {'numFound': 0}}
filters = [q['q'] for q in query['qs'] if q['q']]
filters.extend(self._get_fq(dashboard, query, facet))
if facet:
if facet['type'] == 'nested':
fields_dimensions = [
self._get_dimension_field(f)['name']
for f in self._get_dimension_fields(facet)
]
last_dimension_seen = False
fields = []
for f in reversed(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) AS Count')
last_dimension_seen = True
fields.insert(0,
self._get_dimension_field(f)['select'])
else:
if not last_dimension_seen:
fields.insert(0, self._get_aggregate_function(f))
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) as Count')
fields.insert(0, self._get_dimension_field(facet)['select'])
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s
GROUP BY %(fields_dimensions)s
ORDER BY %(order_by)s
LIMIT %(limit)s''' % {
'database':
database,
'table':
table,
'fields':
', '.join(fields),
'fields_dimensions':
', '.join(fields_dimensions),
'order_by':
', '.join([
self._get_dimension_field(f)['order_by']
for f in self._get_dimension_fields(facet)
]),
'filters':
self._convert_filters_to_where(filters),
'limit':
LIMIT
}
elif facet['type'] == 'function': # 1 dim only now
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s''' % {
'database': database,
'table': table,
'fields': self._get_aggregate_function(facet),
'filters': self._convert_filters_to_where(filters),
}
else:
fields = Collection2.get_field_list(dashboard)
sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
'database':
database,
'table':
table,
'fields':
', '.join(['`%s`' % f if f != '*' else '*' for f in fields])
}
if filters:
sql += ' ' + self._convert_filters_to_where(filters)
sql += ' LIMIT %s' % LIMIT
editor = make_notebook(name='Execute and watch',
editor_type=dashboard['engine'],
statement=sql,
database=database,
status='ready-execute',
skip_historify=True)
response = editor.execute(MockRequest(self.user))
if 'handle' in response and response['handle'].get('sync'):
response['result'] = self._convert_result(response['result'],
dashboard, facet, query)
return response
