def browse(request, name, is_mobile=False):
engine = request.GET.get('engine', 'solr')
collections = get_engine(request.user, engine).datasets()
if not collections and engine == 'solr':
return no_collections(request)
collection = Collection2(user=request.user, name=name, engine=engine)
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
template = 'search.mako'
if is_mobile:
template = 'search_m.mako'
return render(
template, request, {
'collection':
collection,
'query':
query,
'initial':
json.dumps({
'autoLoad':
True,
'collections':
collections,
'layout': [{
"size":
12,
"rows": [{
"widgets": [{
"size": 12,
"name": "Grid Results",
"id": "52f07188-f30f-1296-2450-f77e02e1a5c0",
"widgetType": "resultset-widget",
"properties": {},
"offset": 0,
"isLoading": True,
"klass": "card card-widget span12"
}]
}],
"drops": ["temp"],
"klass":
"card card-home card-column span10"
}],
'is_latest':
LATEST.get(),
'engines':
get_engines(request.user)
}),
'is_owner':
True,
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_index':
can_edit_index(request.user),
'mobile':
is_mobile
})
def new_search(request):
engine = request.GET.get('engine', 'solr')
cluster = request.POST.get('cluster', '""')
collections = get_engine(
request.user, engine,
cluster=cluster).datasets() if engine != 'report' else ['default']
if not collections:
if engine == 'solr':
return no_collections(request)
else:
return importer(request)
collection = Collection2(user=request.user,
name=collections[0],
engine=engine)
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
layout = DEFAULT_LAYOUT if engine != 'report' else REPORT_LAYOUT
if request.GET.get('format', 'plain') == 'json':
return JsonResponse({
'collection': collection.get_props(request.user),
'query': query,
'initial': {
'collections': collections,
'layout': layout,
'qb_layout': QUERY_BUILDER_LAYOUT,
'text_search_layout': TEXT_SEARCH_LAYOUT,
'is_latest': _get_latest(),
'engines': get_engines(request.user)
}
})
else:
return render(
'search.mako', request, {
'collection':
collection,
'query':
query,
'initial':
json.dumps({
'collections': collections,
'layout': layout,
'qb_layout': QUERY_BUILDER_LAYOUT,
'text_search_layout': TEXT_SEARCH_LAYOUT,
'is_latest': _get_latest(),
'engines': get_engines(request.user)
}),
'is_owner':
True,
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_index':
can_edit_index(request.user),
'is_report':
engine == 'report'
})
def index_fields_dynamic(request):
result = {'status': -1, 'message': 'Error'}
try:
name = request.POST['name']
engine = request.POST['engine']
dynamic_fields = get_engine(request.user, engine).luke(name)
result['message'] = ''
result['fields'] = [
Collection2._make_field(name, properties)
for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties
]
result['gridlayout_header_fields'] = [
Collection2._make_gridlayout_header_field({'name': name, 'type': properties.get('type')}, True)
for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties
]
result['status'] = 0
except Exception, e:
result['message'] = force_unicode(e)
def get_collection(request):
result = {'status': -1, 'message': ''}
try:
name = request.POST['name']
engine = request.POST['engine']
collection = Collection2(request.user, name=name, engine=engine)
collection_json = collection.get_json(request.user)
result['collection'] = json.loads(collection_json)
result['status'] = 0
except Exception, e:
result['message'] = force_unicode(e)
def index(request, is_mobile=False, is_embeddable=False):
hue_collections = DashboardController(request.user).get_search_collections()
collection_id = request.GET.get('collection')
if not hue_collections or not collection_id:
return admin_collections(request, True, is_mobile)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception, e:
raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def get_fields(self, collection_or_core_name):
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
try:
field_data = api.fields(collection_or_core_name)
fields = self._format_flags(field_data['schema']['fields'])
except Exception, e:
LOG.warn('/luke call did not succeed: %s' % e)
try:
fields = api.schema_fields(collection_or_core_name)
fields = Collection2._make_luke_from_schema_fields(fields)
except:
LOG.exception(
_('Could not fetch fields for collection %s.') %
collection_or_core_name)
raise PopupException(
_('Could not fetch fields for collection %s. See logs for more info.'
) % collection_or_core_name)
def get_collection(request):
result = {'status': -1, 'message': ''}
try:
name = request.POST.get('name')
engine = request.POST.get('engine')
source = request.POST.get('source')
collection = Collection2(request.user, name=name, engine=engine, source=source)
collection_json = collection.get_json(request.user)
result['collection'] = json.loads(collection_json)
result['status'] = 0
except Exception as e:
result['message'] = force_unicode(e)
return JsonResponse(result)
def new_search(request):
engine = request.GET.get('engine', 'solr')
collections = get_engine(request.user, engine).datasets()
if not collections:
return no_collections(request)
collection = Collection2(user=request.user,
name=collections[0],
engine=engine)
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
if request.GET.get('format', 'plain') == 'json':
return JsonResponse({
'collection': collection.get_props(request.user),
'query': query,
'initial': {
'collections': collections,
'layout': DEFAULT_LAYOUT,
'is_latest': LATEST.get(),
'engines': get_engines(request.user)
}
})
else:
return render(
'search.mako', request, {
'collection':
collection,
'query':
query,
'initial':
json.dumps({
'collections': collections,
'layout': DEFAULT_LAYOUT,
'is_latest': LATEST.get(),
'engines': get_engines(request.user)
}),
'is_owner':
True,
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_index':
can_edit_index(request.user)
})
def copy_collections(self, collection_ids):
result = {'status': -1, 'message': ''}
try:
for doc2 in self.get_shared_search_collections():
if doc2.id in collection_ids:
doc2 = Document2.objects.get_by_uuid(user=self.user, uuid=doc2.uuid)
doc = doc2.doc.get()
name = doc2.name + '-copy'
doc2 = doc2.copy(name=name, owner=self.user)
doc.copy(content_object=doc2, name=name, owner=self.user)
collection = Collection2(self.user, document=doc2)
collection.data['collection']['label'] = name
doc2.update_data({'collection': collection.data['collection']})
doc2.save()
result['status'] = 0
except Exception, e:
LOG.exception('Error copying collection')
result['message'] = unicode(str(e), "utf8")
def query(self, dashboard, query, facet=None):
if query['qs'] == [{'q': '_root_:*'}]:
return {'response': {'numFound': 0}}
filters = [q['q'] for q in query['qs'] if q['q']]
filters.extend(self._get_fq(dashboard, query, facet))
result_properties = {}
timeFilter = self._get_time_filter_query(dashboard, query)
if timeFilter:
filters.append(timeFilter)
if self.source == 'query':
sql_from = '(%(query)s) t' % {
'query': self._get_query(dashboard['name'])
}
database, table = '', ''
else:
database, table = self._get_database_table_names(dashboard['name'])
sql_from = '%(backticks)s%(database)s%(backticks)s.%(backticks)s%(table)s%(backticks)s' % {
'database': database,
'table': table,
'backticks': self.backticks
}
if facet and facet['properties']['facets']:
for i, _facet in enumerate(facet['properties']['facets']):
_facet['position'] = i
if facet['type'] == 'nested':
fields_dimensions = [
self._get_dimension_field(f)['name']
for f in self._get_dimension_fields(facet)
]
last_dimension_seen = False
fields = []
for f in reversed(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
if not last_dimension_seen:
fields.insert(
0, 'COUNT(*) AS %(field)s_%(position)s' % f)
last_dimension_seen = True
fields.insert(
0,
self._get_dimension_field(f)
['name' if last_dimension_seen else 'select'])
else:
if not last_dimension_seen:
fields.insert(
0,
self._get_aggregate_function(f) +
'AS %(field)s_%(position)s' % f)
has_facet_mincount_greater_than_one = [
f for f in facet['properties']['facets']
if f['mincount'] > 1
]
if has_facet_mincount_greater_than_one and self._supports_count_over(
):
mincount_fields_name = []
mincount_fields_operation = []
mincount_where = []
for f in facet['properties']['facets']:
mincount_fields_name.append(f['field'])
mincount_field_name = 'count__' + '_'.join(
mincount_fields_name)
mincount_fields_operation.append(
'COUNT(*) OVER (PARTITION BY %s) AS %s' %
(', '.join(mincount_fields_name),
mincount_field_name))
mincount_where.append(
'%s >= %s' %
(mincount_field_name, str(f['mincount'])))
sql_from = '''(SELECT * FROM (SELECT *, %(fields)s
FROM %(sql_from)s) default
WHERE %(where)s) default''' % {
'fields': ', '.join(mincount_fields_operation),
'sql_from': sql_from,
'where': ' AND '.join(mincount_where)
}
order_by = ', '.join([
self._get_dimension_field(f)['order_by']
for f in reversed(facet['properties']['facets'])
if f['sort'] != 'default'
])
sql = '''SELECT %(fields)s
FROM %(sql_from)s
%(filters)s
GROUP BY %(fields_dimensions)s
%(order_by)s
LIMIT %(limit)s''' % {
'sql_from': sql_from,
'fields': ', '.join(fields),
'fields_dimensions': ', '.join(fields_dimensions),
'order_by': 'ORDER BY %s' % order_by if order_by else '',
'filters': self._convert_filters_to_where(filters),
'limit': LIMIT
}
elif facet['type'] == 'function': # 1 dim only now
aggregate_function = facet['properties']['facets'][0][
'aggregate']['function']
if (aggregate_function == 'percentile' or aggregate_function
== 'median') and not self._supports_percentile(
) and self._supports_cume_dist():
sql_from = '''
(SELECT *
FROM
(
SELECT %(field)s, cume_dist() OVER (ORDER BY %(field)s) * 100 AS cume_dist__%(field)s
FROM %(sql_from)s
) DEFAULT
WHERE cume_dist__%(field)s >= %(value)s) DEFAULT
''' % {
'field':
facet['properties']['facets'][0]['field'],
'value':
facet['properties']['facets'][0]['aggregate']
['percentile']
if aggregate_function == 'percentile' else 50,
'sql_from':
sql_from,
}
sql = '''SELECT %(fields)s
FROM %(sql_from)s
%(filters)s''' % {
'sql_from':
sql_from,
'fields':
self._get_aggregate_function(
facet['properties']['facets'][0]),
'filters':
self._convert_filters_to_where(filters),
}
elif facet['type'] == 'statement':
doc = Document2.objects.get_by_uuid(
user=self.user,
uuid=facet['properties']['statementUuid'],
perm_type='read')
snippets = doc.data_dict.get('snippets', [])
sql = snippets[0].get('statement', '')
result_properties = facet['properties']['result']
else:
fields = Collection2.get_field_list(dashboard)
order_by = ', '.join([
'%(backticks)s%(name)s%(backticks)s %(direction)s' % {
'backticks': self.backticks,
'name': f['name'],
'direction': f['sort']['direction']
} for f in dashboard['template']['fieldsAttributes']
if f['sort']['direction'] and f['name'] in fields
])
sql = '''
SELECT %(fields)s
FROM %(sql_from)s
%(filters)s
%(order_by)s
%(limit)s''' % {
'sql_from':
sql_from,
'fields':
', '.join([
'%(backticks)s%(column)s%(backticks)s as %(backticks)s%(column)s%(backticks)s'
% {
'backticks': self.backticks,
'column': f
} if f != '*' else '*' for f in fields
]),
'filters':
self._convert_filters_to_where(filters) if filters else '',
'order_by':
'ORDER BY %s' % order_by if order_by else '',
'limit':
'LIMIT %s' % dashboard['template']['rows'] or LIMIT
}
editor = make_notebook(name='Execute and watch',
editor_type=dashboard['engine'],
statement=sql,
database=database,
status='ready-execute',
skip_historify=True,
result_properties=result_properties)
response = editor.execute(MockRequest(self.user, self.cluster))
if 'handle' in response and response['handle'].get('sync'):
response['result'] = self._convert_result(response['result'],
dashboard, facet, query)
return response
def luke(self, collection):
fields = self.schema_fields(collection)
return {'fields': Collection2._make_luke_from_schema_fields(fields)}
def query(self, dashboard, query, facet=None):
database, table = self._get_database_table_names(dashboard['name'])
if query['qs'] == [{'q': '_root_:*'}]:
return {'response': {'numFound': 0}}
filters = [q['q'] for q in query['qs'] if q['q']]
filters.extend(self._get_fq(dashboard, query, facet))
timeFilter = self._get_time_filter_query(dashboard, query)
if timeFilter:
filters.append(timeFilter)
if facet:
if facet['type'] == 'nested':
fields_dimensions = [self._get_dimension_field(f)['name'] for f in self._get_dimension_fields(facet)]
last_dimension_seen = False
fields = []
for f in reversed(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) AS Count')
last_dimension_seen = True
fields.insert(0, self._get_dimension_field(f)['select'])
else:
if not last_dimension_seen:
fields.insert(0, self._get_aggregate_function(f))
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) as Count')
fields.insert(0, self._get_dimension_field(facet)['select'])
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s
GROUP BY %(fields_dimensions)s
ORDER BY %(order_by)s
LIMIT %(limit)s''' % {
'database': database,
'table': table,
'fields': ', '.join(fields),
'fields_dimensions': ', '.join(fields_dimensions),
'order_by': ', '.join([self._get_dimension_field(f)['order_by'] for f in self._get_dimension_fields(facet)]),
'filters': self._convert_filters_to_where(filters),
'limit': LIMIT
}
elif facet['type'] == 'function': # 1 dim only now
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s''' % {
'database': database,
'table': table,
'fields': self._get_aggregate_function(facet),
'filters': self._convert_filters_to_where(filters),
}
else:
fields = Collection2.get_field_list(dashboard)
sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
'database': database,
'table': table,
'fields': ', '.join(['`%s`' % f if f != '*' else '*' for f in fields])
}
if filters:
sql += ' ' + self._convert_filters_to_where(filters)
sql += ' LIMIT %s' % LIMIT
editor = make_notebook(
name='Execute and watch',
editor_type=dashboard['engine'],
statement=sql,
database=database,
status='ready-execute',
skip_historify=True
)
response = editor.execute(MockRequest(self.user))
if 'handle' in response and response['handle'].get('sync'):
response['result'] = self._convert_result(response['result'], dashboard, facet, query)
return response
def index(request, is_mobile=False):
engine = request.GET.get('engine', 'solr')
cluster = request.POST.get('cluster', '""')
collection_id = request.GET.get('collection')
collections = get_engine(
request.user, engine,
cluster=cluster).datasets() if engine != 'report' else ['default']
if not collections:
if engine == 'solr':
return no_collections(request)
else:
return importer(request)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception as e:
raise PopupException(
e,
title=
_("Dashboard does not exist or you don't have the permission to access it."
))
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
if request.method == 'GET':
if 'q' in request.GET:
query['qs'][0]['q'] = antixss(request.GET.get('q', ''))
if 'qd' in request.GET:
query['qd'] = antixss(request.GET.get('qd', ''))
template = 'search.mako'
if is_mobile:
template = 'search_m.mako'
engine = collection.data['collection'].get('engine', 'solr')
return render(
template, request, {
'collection':
collection,
'query':
json.dumps(query),
'initial':
json.dumps({
'collections': collections,
'layout': DEFAULT_LAYOUT,
'qb_layout': QUERY_BUILDER_LAYOUT,
'text_search_layout': TEXT_SEARCH_LAYOUT,
'is_latest': _get_latest(),
'engines': get_engines(request.user)
}),
'is_owner':
collection_doc.can_write(request.user) if USE_NEW_EDITOR.get() else
collection_doc.doc.get().can_write(request.user),
'can_edit_index':
can_edit_index(request.user),
'is_embeddable':
request.GET.get('is_embeddable', False),
'mobile':
is_mobile,
'is_report':
collection.data['collection'].get('engine') == 'report'
})
def _create_facet(collection, user, facet_id, facet_label, facet_field,
widget_type, window_size):
properties = {
'sort': 'desc',
'canRange': False,
'stacked': False,
'limit': 10,
'mincount': 0,
'missing': False,
'isDate': False,
'slot': 0,
'aggregate': {
'function': 'unique',
'formula': '',
'plain_formula': '',
'percentile': 50
}
}
template = {
"showFieldList": True,
"showGrid": False,
"showChart": True,
"chartSettings": {
'chartType':
'pie' if widget_type == 'pie2-widget' else
('timeline' if widget_type == 'timeline-widget' else
('gradientmap' if widget_type == 'gradient-map-widget' else 'bars'
)),
'chartSorting':
'none',
'chartScatterGroup':
None,
'chartScatterSize':
None,
'chartScope':
'world',
'chartX':
None,
'chartYSingle':
None,
'chartYMulti': [],
'chartData': [],
'chartMapLabel':
None,
'chartSelectorType':
'bar'
},
"fieldsAttributes": [],
"fieldsAttributesFilter": "",
"filteredAttributeFieldsAll": True,
"fields": [],
"fieldsSelected": [],
"leafletmap": {
'latitudeField': None,
'longitudeField': None,
'labelField': None
}, # Use own?
'leafletmapOn': False,
'isGridLayout': False,
"hasDataForChart": True,
"rows": 25,
}
if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'):
facet_type = 'pivot'
elif widget_type == 'document-widget':
# SQL query, 1 solr widget
if collection['selectedDocument'].get('uuid'):
doc = Document2.objects.get_by_uuid(
user=user,
uuid=collection['selectedDocument']['uuid'],
perm_type='read')
snippets = doc.data_dict.get('snippets', [])
if snippets:
table_metadata = get_api(MockRequest(user, '""'),
snippets[0]).autocomplete(
{
'source': 'query',
'type': snippets[0]['type']
}, doc.id)
template['fieldsAttributes'] = [
Collection2._make_gridlayout_header_field(field)
for field in table_metadata['extended_columns']
]
properties['engine'] = snippets[0]['type']
if snippets[0]['result']['handle']:
handle = snippets[0]['result']['handle']
# Replace previous_statement_hash so that we rerun current statement
properties['result'] = {
'handle': {
'statement_id':
handle['statement_id'],
'statements_count':
handle['statements_count'],
'previous_statement_hash':
hashlib.sha224(str(uuid.uuid4())).hexdigest()
}
}
else:
properties['result'] = {'handle': {}}
properties['statement'] = snippets[0]['statement_raw']
else:
properties['result'] = {'handle': {}}
properties['statement'] = ''
else: # Demo data for now
properties['statement'] = 'select * from customers'
properties['result'] = {'handle': {}}
properties['uuid'] = facet_field
properties['facets'] = [{
'canRange': False,
'field': 'blank',
'limit': 10,
'mincount': 0,
'sort': 'desc',
'aggregate': {
'function': 'count'
},
'isDate': False,
'type': 'field'
}]
facet_type = 'statement'
else:
api = get_engine(user, collection)
range_properties = _new_range_facet(api, collection, facet_field,
widget_type, window_size)
if range_properties:
facet_type = 'range'
properties.update(range_properties)
properties['initial_gap'] = properties['gap']
properties['initial_start'] = properties['start']
properties['initial_end'] = properties['end']
else:
facet_type = 'field'
if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget',
'tree2-widget', 'text-facet-widget', 'hit-widget',
'gradient-map-widget'):
# properties = {'canRange': False, 'stacked': False, 'limit': 10} # TODO: Lighter weight top nested facet
properties['facets_form'] = NESTED_FACET_FORM
# Not supported on dim 2 currently
properties['facets_form']['type'] = 'field'
properties['facets_form']['canRange'] = False
properties['facets_form']['isFacetForm'] = True
facet = NESTED_FACET_FORM.copy()
facet['field'] = facet_field
facet['limit'] = 10
facet['fieldLabel'] = facet_field
facet['multiselect'] = True
if widget_type == 'text-facet-widget':
properties['type'] = facet_type
if USE_GRIDSTER.get():
properties['limit'] = facet['limit'] = 100
if range_properties:
# TODO: timeline still uses properties from top properties
facet.update(range_properties)
facet['initial_gap'] = facet['gap']
facet['initial_start'] = facet['start']
facet['initial_end'] = facet['end']
facet['stacked'] = False
facet['type'] = 'range'
else:
facet['type'] = facet_type
if collection.get('engine', 'solr') != 'solr':
facet['sort'] = 'default'
properties['facets'] = [facet]
properties['domain'] = {'blockParent': [], 'blockChildren': []}
properties['compare'] = COMPARE_FACET
properties['filter'] = QUERY_FACET
if widget_type == 'hit-widget':
facet_type = 'function'
facet['aggregate']['function'] = 'unique'
else:
facet_type = 'nested'
facet['aggregate']['function'] = 'count'
if widget_type == 'pie2-widget':
properties['scope'] = 'stack'
properties['timelineChartType'] = 'bar'
elif widget_type == 'tree2-widget':
properties['scope'] = 'tree'
properties['facets_form']['limit'] = 5
properties['isOldPivot'] = True
elif widget_type == 'gradient-map-widget':
properties['scope'] = 'world'
facet['limit'] = 100
else:
properties['scope'] = 'stack'
properties['timelineChartType'] = 'bar'
if widget_type in ('tree-widget', 'heatmap-widget',
'map-widget') and widget_type != 'gradient-map-widget':
properties['mincount'] = 1
properties['facets'] = []
properties['stacked'] = True
properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
if widget_type == 'map-widget':
properties['scope'] = 'world'
properties['limit'] = 100
else:
properties[
'scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree'
if widget_type == 'histogram-widget':
properties['enableSelection'] = True
properties['timelineChartType'] = 'bar'
properties['extraSeries'] = []
return {
'id': facet_id,
'label': facet_label,
'field': facet_field,
'type': facet_type,
'widgetType': widget_type,
'properties': properties,
# Hue 4+
'template': template,
'queryResult': {}
}
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
params = self._get_params() + (
('q', self._get_q(query)),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
json_facets = {}
timeFilter = self._get_range_borders(collection, query)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']), )
elif facet['type'] == 'range' or facet['type'] == 'range-up':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
'mincount': int(facet['properties']['mincount'])
}
if timeFilter and timeFilter['time_field'] == facet[
'field'] and (
facet['id']
not in timeFilter['time_filter_overrides']
or facet['widgetType'] != 'histogram-widget'):
keys.update(
self._get_time_filter_query(timeFilter, facet))
params += ((
'facet.range',
'{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s'
% keys), )
elif facet['type'] == 'field':
keys = {
'id':
'%(id)s' % facet,
'field':
facet['field'],
'key':
'%(field)s-%(id)s' % facet,
'limit':
int(facet['properties'].get('limit', 10)) +
(1 if facet['widgetType'] == 'facet-widget' else 0),
'mincount':
int(facet['properties']['mincount'])
}
params += ((
'facet.field',
'{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s'
% keys), )
elif facet['type'] == 'nested':
_f = {
'field':
facet['field'],
'limit':
int(facet['properties'].get('limit', 10)) +
(1
if facet['widgetType'] == 'text-facet-widget' else 0),
'mincount':
int(facet['properties']['mincount']),
'sort': {
'count': facet['properties']['sort']
},
}
if facet['properties']['domain'].get(
'blockParent'
) or facet['properties']['domain'].get('blockChildren'):
_f['domain'] = {}
if facet['properties']['domain'].get('blockParent'):
_f['domain']['blockParent'] = ' OR '.join(
facet['properties']['domain']['blockParent'])
if facet['properties']['domain'].get('blockChildren'):
_f['domain']['blockChildren'] = ' OR '.join(
facet['properties']['domain']['blockChildren'])
if 'start' in facet['properties'] and not facet[
'properties'].get('type') == 'field':
_f.update({
'type': 'range',
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
})
if timeFilter and timeFilter['time_field'] == facet[
'field'] and (
facet['id']
not in timeFilter['time_filter_overrides']
or facet['widgetType'] != 'bucket-widget'):
_f.update(
self._get_time_filter_query(timeFilter, facet))
else:
_f.update({
'type': 'terms',
'field': facet['field'],
'excludeTags': facet['id'],
'offset': 0,
'numBuckets': True,
'allBuckets': True,
#'prefix': '' # Forbidden on numeric fields
})
if facet['properties']['canRange'] and not facet[
'properties']['isDate']:
del _f['mincount'] # Numeric fields do not support
if facet['properties']['facets']:
self._n_facet_dimension(facet, _f,
facet['properties']['facets'],
1)
if facet['widgetType'] == 'text-facet-widget':
_fname = _f['facet'].keys()[0]
_f['sort'] = {_fname: facet['properties']['sort']}
# domain = '-d2:NaN' # Solr 6.4
json_facets[facet['id']] = _f
elif facet['type'] == 'function':
json_facets[facet['id']] = self._get_aggregate_function(
facet)
json_facets['processEmpty'] = True
elif facet['type'] == 'pivot':
if facet['properties']['facets'] or facet[
'widgetType'] == 'map-widget':
fields = facet['field']
fields_limits = []
for f in facet['properties']['facets']:
fields_limits.append('f.%s.facet.limit=%s' %
(f['field'], f['limit']))
fields_limits.append('f.%s.facet.mincount=%s' %
(f['field'], f['mincount']))
fields += ',' + f['field']
keys = {
'id': '%(id)s' % facet,
'key': '%(field)s-%(id)s' % facet,
'field': facet['field'],
'fields': fields,
'limit': int(facet['properties'].get('limit', 10)),
'mincount': int(facet['properties']['mincount']),
'fields_limits': ' '.join(fields_limits)
}
params += ((
'facet.pivot',
'{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s'
% keys), )
if json_facets:
params += (('json.facet', json.dumps(json_facets)), )
params += self._get_fq(collection, query)
from dashboard.models import Collection2
fl = urllib.unquote(
utf_quoter(','.join(Collection2.get_field_list(collection))))
nested_fields = self._get_nested_fields(collection)
if nested_fields:
fl += urllib.unquote(
utf_quoter(',[child parentFilter="%s"]' %
' OR '.join(nested_fields)))
params += (('fl', fl), )
params += (
('hl', 'true'),
('hl.fl', '*'),
('hl.snippets', 5),
('hl.fragsize', 1000),
)
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(
lambda attribute: field == attribute['name'],
collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append(
'%s %s' %
(field, attribute_field[0]['sort']['direction']))
if fields:
params += (('sort', ','.join(fields)), )
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def luke(self, collection):
fields = self.schema_fields(collection)
return {'fields': Collection2._make_luke_from_schema_fields(fields)}
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
params = self._get_params() + (
('q', self._get_q(query)),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
json_facets = {}
timeFilter = self._get_range_borders(collection, query)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']),)
elif facet['type'] == 'range' or facet['type'] == 'range-up':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
'mincount': int(facet['properties']['mincount'])
}
if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'):
keys.update(self._get_time_filter_query(timeFilter, facet))
params += (
('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'field':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0),
'mincount': int(facet['properties']['mincount'])
}
params += (
('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'nested':
_f = {}
if facet['properties']['facets']:
self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1, timeFilter)
if facet['properties'].get('domain'):
if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'):
_f['domain'] = {}
if facet['properties']['domain'].get('blockParent'):
_f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent'])
if facet['properties']['domain'].get('blockChildren'):
_f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren'])
if _f:
sort = {'count': facet['properties']['facets'][0]['sort']}
for i, agg in enumerate(self._get_dimension_aggregates(facet['properties']['facets'][1:])):
if agg['sort'] != 'default':
agg_function = self._get_aggregate_function(agg)
sort = {'agg_%02d_%02d:%s' % (1, i, agg_function): agg['sort']}
if sort.get('count') == 'default':
sort['count'] = 'desc'
dim_key = [key for key in _f['facet'].keys() if 'dim' in key][0]
_f['facet'][dim_key].update({
'excludeTags': facet['id'],
'offset': 0,
'numBuckets': True,
'allBuckets': True,
'sort': sort
#'prefix': '' # Forbidden on numeric fields
})
json_facets[facet['id']] = _f['facet'][dim_key]
elif facet['type'] == 'function':
if facet['properties']['facets']:
json_facets[facet['id']] = self._get_aggregate_function(facet['properties']['facets'][0])
if facet['properties']['compare']['is_enabled']:
# TODO: global compare override
unit = re.split('\d+', facet['properties']['compare']['gap'])[1]
json_facets[facet['id']] = {
'type': 'range',
'field': collection['timeFilter'].get('field'),
'start': 'NOW/%s-%s-%s' % (unit, facet['properties']['compare']['gap'], facet['properties']['compare']['gap']),
'end': 'NOW/%s' % unit,
'gap': '+%(gap)s' % facet['properties']['compare'],
'facet': {facet['id']: json_facets[facet['id']]}
}
if facet['properties']['filter']['is_enabled']:
json_facets[facet['id']] = {
'type': 'query',
'q': facet['properties']['filter']['query'] or EMPTY_QUERY.get(),
'facet': {facet['id']: json_facets[facet['id']]}
}
json_facets['processEmpty'] = True
elif facet['type'] == 'pivot':
if facet['properties']['facets'] or facet['widgetType'] == 'map-widget':
fields = facet['field']
fields_limits = []
for f in facet['properties']['facets']:
fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit']))
fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount']))
fields += ',' + f['field']
keys = {
'id': '%(id)s' % facet,
'key': '%(field)s-%(id)s' % facet,
'field': facet['field'],
'fields': fields,
'limit': int(facet['properties'].get('limit', 10)),
'mincount': int(facet['properties']['mincount']),
'fields_limits': ' '.join(fields_limits)
}
params += (
('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys),
)
if json_facets:
params += (
('json.facet', json.dumps(json_facets)),
)
params += self._get_fq(collection, query)
fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection))))
nested_fields = self._get_nested_fields(collection)
if nested_fields:
fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields)))
if collection['template']['moreLikeThis'] and fl != ['*']: # Potential conflict with nested documents
id_field = collection.get('idField', 'id')
params += (
('mlt', 'true'),
('mlt.fl', fl.replace(',%s' % id_field, '')),
('mlt.mintf', 1),
('mlt.mindf', 1),
('mlt.maxdf', 50),
('mlt.maxntp', 1000),
('mlt.count', 10),
#('mlt.minwl', 1),
#('mlt.maxwl', 1),
)
fl = '*'
params += (('fl', fl),)
params += (
('hl', 'true'),
('hl.fl', '*'),
('hl.snippets', 5),
('hl.fragsize', 1000),
)
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
if fields:
params += (
('sort', ','.join(fields)),
)
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def query(self, dashboard, query, facet=None):
database, table = self._get_database_table_names(dashboard['name'])
if query['qs'] == [{'q': '_root_:*'}]:
return {'response': {'numFound': 0}}
filters = [q['q'] for q in query['qs'] if q['q']]
filters.extend(self._get_fq(dashboard, query, facet))
timeFilter = self._get_time_filter_query(dashboard, query)
if timeFilter:
filters.append(timeFilter)
if facet:
if facet['type'] == 'nested':
fields_dimensions = [
self._get_dimension_field(f)['name']
for f in self._get_dimension_fields(facet)
]
last_dimension_seen = False
fields = []
for f in reversed(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) AS Count')
last_dimension_seen = True
fields.insert(0,
self._get_dimension_field(f)['select'])
else:
if not last_dimension_seen:
fields.insert(0, self._get_aggregate_function(f))
if not last_dimension_seen:
fields.insert(0, 'COUNT(*) as Count')
fields.insert(0, self._get_dimension_field(facet)['select'])
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s
GROUP BY %(fields_dimensions)s
ORDER BY %(order_by)s
LIMIT %(limit)s''' % {
'database':
database,
'table':
table,
'fields':
', '.join(fields),
'fields_dimensions':
', '.join(fields_dimensions),
'order_by':
', '.join([
self._get_dimension_field(f)['order_by']
for f in self._get_dimension_fields(facet)
]),
'filters':
self._convert_filters_to_where(filters),
'limit':
LIMIT
}
elif facet['type'] == 'function': # 1 dim only now
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s''' % {
'database': database,
'table': table,
'fields': self._get_aggregate_function(facet),
'filters': self._convert_filters_to_where(filters),
}
else:
fields = Collection2.get_field_list(dashboard)
sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
'database':
database,
'table':
table,
'fields':
', '.join(['`%s`' % f if f != '*' else '*' for f in fields])
}
if filters:
sql += ' ' + self._convert_filters_to_where(filters)
sql += ' LIMIT %s' % LIMIT
editor = make_notebook(name='Execute and watch',
editor_type=dashboard['engine'],
statement=sql,
database=database,
status='ready-execute',
skip_historify=True)
response = editor.execute(MockRequest(self.user))
if 'handle' in response and response['handle'].get('sync'):
response['result'] = self._convert_result(response['result'],
dashboard, facet, query)
return response
def copy_document(request):
uuid = json.loads(request.POST.get('uuid'), '""')
if not uuid:
raise PopupException(_('copy_document requires uuid'))
# Document2 and Document model objects are linked and both are saved when saving
document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
# Document model object
document1 = document.doc.get()
if document.type == 'directory':
raise PopupException(_('Directory copy is not supported'))
name = document.name + '-copy'
# Make the copy of the Document2 model object
copy_document = document.copy(name=name, owner=request.user)
# Make the copy of Document model object too
document1.copy(content_object=copy_document, name=name, owner=request.user)
# Import workspace for all oozie jobs
if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace
# Update the name field in the json 'data' field
if document.type == 'oozie-workflow2':
workflow = Workflow(document=document)
workflow.update_name(name)
workflow.update_uuid(copy_document.uuid)
_import_workspace(request.fs, request.user, workflow)
copy_document.update_data(
{'workflow': workflow.get_data()['workflow']})
copy_document.save()
if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
if document.type == 'oozie-bundle2':
bundle_or_coordinator = Bundle(document=document)
else:
bundle_or_coordinator = Coordinator(document=document)
json_data = bundle_or_coordinator.get_data_for_json()
json_data['name'] = name
json_data['uuid'] = copy_document.uuid
copy_document.update_data(json_data)
copy_document.save()
_import_workspace(request.fs, request.user, bundle_or_coordinator)
elif document.type == 'search-dashboard':
from dashboard.models import Collection2
collection = Collection2(request.user, document=document)
collection.data['collection']['label'] = name
collection.data['collection']['uuid'] = copy_document.uuid
copy_document.update_data(
{'collection': collection.data['collection']})
copy_document.save()
# Keep the document and data in sync
else:
copy_data = copy_document.data_dict
if 'name' in copy_data:
copy_data['name'] = name
if 'uuid' in copy_data:
copy_data['uuid'] = copy_document.uuid
copy_document.update_data(copy_data)
copy_document.save()
return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def query(self, dashboard, query, facet=None):
database, table = self._get_database_table_names(dashboard['name'])
if query['qs'] == [{'q': '_root_:*'}]:
return {'response': {'numFound': 0}}
filters = [q['q'] for q in query['qs'] if q['q']]
filters.extend(self._get_fq(dashboard, query, facet))
timeFilter = self._get_time_filter_query(dashboard, query)
if timeFilter:
filters.append(timeFilter)
if facet and facet['properties']['facets']:
for i, _facet in enumerate(facet['properties']['facets']):
_facet['position'] = i
if facet['type'] == 'nested':
fields_dimensions = [
self._get_dimension_field(f)['name']
for f in self._get_dimension_fields(facet)
]
last_dimension_seen = False
fields = []
for f in reversed(facet['properties']['facets']):
if f['aggregate']['function'] == 'count':
if not last_dimension_seen:
fields.insert(
0, 'COUNT(*) AS %(field)s_%(position)s' % f)
last_dimension_seen = True
fields.insert(
0,
self._get_dimension_field(f)
['name' if last_dimension_seen else 'select'])
else:
if not last_dimension_seen:
fields.insert(
0,
self._get_aggregate_function(f) +
'AS %(field)s_%(position)s' % f)
if self._supports_count_over():
mincount_fields_name = []
mincount_fields_operation = []
mincount_where = []
for f in facet['properties']['facets']:
mincount_fields_name.append(f['field'])
mincount_field_name = '_'.join(
mincount_fields_name) + "_count"
mincount_fields_operation.append(
'count(*) over (partition by %s) as %s' %
(','.join(mincount_fields_name),
mincount_field_name))
mincount_where.append(
'%s >= %s' %
(mincount_field_name, str(f['mincount'])))
sql_mincount = '''(SELECT * FROM (SELECT *,%(fields)s
FROM %(database)s.%(table)s) default
WHERE %(where)s) default''' % {
'fields': ', '.join(mincount_fields_operation),
'database': database,
'table': table,
'where': ' and '.join(mincount_where)
}
else:
sql_mincount = '(%(database)s.%(table)s)' % {
'database': database,
'table': table
}
order_by = ', '.join([
self._get_dimension_field(f)['order_by']
for f in reversed(facet['properties']['facets'])
if f['sort'] != 'default'
])
sql = '''SELECT %(fields)s
FROM %(sql_mincount)s
%(filters)s
GROUP BY %(fields_dimensions)s
%(order_by)s
LIMIT %(limit)s''' % {
'sql_mincount': sql_mincount,
'database': database,
'table': table,
'fields': ', '.join(fields),
'fields_dimensions': ', '.join(fields_dimensions),
'order_by': 'ORDER BY %s' % order_by if order_by else '',
'filters': self._convert_filters_to_where(filters),
'limit': LIMIT
}
elif facet['type'] == 'function': # 1 dim only now
sql = '''SELECT %(fields)s
FROM %(database)s.%(table)s
%(filters)s''' % {
'database':
database,
'table':
table,
'fields':
self._get_aggregate_function(
facet['properties']['facets'][0]),
'filters':
self._convert_filters_to_where(filters),
}
else:
fields = Collection2.get_field_list(dashboard)
sql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
'database':
database,
'table':
table,
'fields':
', '.join(['`%s`' % f if f != '*' else '*' for f in fields])
}
if filters:
sql += ' ' + self._convert_filters_to_where(filters)
sql += ' LIMIT %s' % dashboard['template']['rows'] or LIMIT
editor = make_notebook(name='Execute and watch',
editor_type=dashboard['engine'],
statement=sql,
database=database,
status='ready-execute',
skip_historify=True)
response = editor.execute(MockRequest(self.user))
if 'handle' in response and response['handle'].get('sync'):
response['result'] = self._convert_result(response['result'],
dashboard, facet, query)
return response
