def _create_solr_cloud_collection(self, name, fields, unique_key_field,
df):
client = SolrClient(self.user)
with ZookeeperClient(hosts=client.get_zookeeper_host(),
read_only=False) as zc:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
tmp_path, solr_config_path = copy_configs(
fields=fields,
unique_key_field=unique_key_field,
df=df,
solr_cloud_mode=client.is_solr_cloud_mode(),
is_solr_six_or_more=client.is_solr_six_or_more(),
is_solr_hdfs_mode=client.is_solr_with_hdfs())
try:
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(
_('Error in copying Solr configurations: %s') % e)
finally:
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def get_range_facet(request):
result = {'status': -1, 'message': ''}
try:
collection = json.loads(request.POST.get('collection', '{}')) # Perms
facet = json.loads(request.POST.get('facet', '{}'))
action = request.POST.get('action', 'select')
solr_api = SolrApi(SOLR_URL.get(), request.user)
if action == 'select':
properties = _guess_gap(solr_api, collection, facet,
facet['properties']['start'],
facet['properties']['end'])
else:
properties = _zoom_range_facet(solr_api, collection, facet)
result['properties'] = properties
result['status'] = 0
except Exception, e:
result['message'] = unicode(str(e), "utf8")
def get_stats(request):
result = {'status': -1, 'message': 'Error'}
try:
collection = json.loads(request.POST.get('collection', '{}'))
query = json.loads(request.POST.get('query', '{}'))
analysis = json.loads(request.POST.get('analysis', '{}'))
field = analysis['name']
facet = analysis['stats']['facet']
result['stats'] = SolrApi(SOLR_URL.get(),
request.user).stats(collection['name'],
[field], query, facet)
result['status'] = 0
result['message'] = ''
except Exception, e:
result['message'] = force_unicode(e)
if 'not currently supported' in result['message']:
result['status'] = 1
result['message'] = _('This field does not support stats')
def get_terms(request):
result = {'status': -1, 'message': 'Error'}
try:
collection = json.loads(request.POST.get('collection', '{}'))
analysis = json.loads(request.POST.get('analysis', '{}'))
limit = json.loads(request.POST.get('limit', '25'))
support_distributed = [
engine for engine in get_engines(request.user)
if engine['type'] == 'solr'
][0]['analytics']
field = analysis['name']
properties = {
'terms.limit': limit,
'terms.distrib': str(support_distributed).lower(),
# lower
# mincount
# maxcount
}
if analysis['terms']['prefix']:
properties['terms.regex'] = '.*%(prefix)s.*' % analysis[
'terms'] # Use regexp instead of case sensitive 'terms.prefix'
properties['terms.regex.flag'] = 'case_insensitive'
result['terms'] = SolrApi(SOLR_URL.get(),
request.user).terms(collection['name'],
field, properties)
result['terms'] = pairwise2(field, [], result['terms']['terms'][field])
result['status'] = 0
result['message'] = ''
except Exception, e:
result['message'] = force_unicode(e)
if 'not currently supported' in result['message']:
result['status'] = 1
result['message'] = _('This field does not support stats')
def index_fields_dynamic(request):
result = {'status': -1, 'message': 'Error'}
try:
name = request.POST['name']
dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name)
result['message'] = ''
result['fields'] = [
Collection2._make_field(name, properties)
for name, properties in dynamic_fields['fields'].iteritems()
if 'dynamicBase' in properties
]
result['gridlayout_header_fields'] = [
Collection2._make_gridlayout_header_field({'name': name}, True)
for name, properties in dynamic_fields['fields'].iteritems()
if 'dynamicBase' in properties
]
result['status'] = 0
except Exception, e:
result['message'] = unicode(str(e), "utf8")
def search(request):
response = {}
collection = json.loads(request.POST.get('collection', '{}'))
query = json.loads(request.POST.get('query', '{}'))
query['download'] = 'download' in request.POST
if collection:
try:
response = SolrApi(SOLR_URL.get(),
request.user).query(collection, query)
response = augment_solr_response(response, collection, query)
except RestException, e:
try:
response['error'] = json.loads(e.message)['error']['msg']
except:
LOG.exception('failed to parse json response')
response['error'] = force_unicode(e)
except Exception, e:
raise PopupException(e, title=_('Error while accessing Solr'))
response['error'] = force_unicode(e)
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
client = SolrClient(self.user)
if core:
raise PopupException(_('Cannot remove Solr cores.'))
if api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
with ZookeeperClient(hosts=client.get_zookeeper_host(),
read_only=False) as zc:
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
api.add_collection(name)
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
def update_collection(self, name, fields):
"""
Only create new fields
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
# Create only new fields
# Fields that already exist, do not overwrite since there is no way to do that, currently.
old_field_names = list(api.fields(name)['schema']['fields'].keys())
new_fields = [
field for field in fields if field['name'] not in old_field_names
]
new_fields_filtered = []
for field in new_fields:
new_field = {}
for attribute in [
attribute for attribute in ALLOWED_FIELD_ATTRIBUTES
if attribute in field
]:
new_field[attribute] = field[attribute]
new_fields_filtered.append(new_field)
api.add_fields(name, new_fields_filtered)
def query_suggest(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
collection = json.loads(request.POST.get('collection', '{}'))
query = request.POST.get('query', '')
result = {'status': -1, 'message': ''}
solr_query = {}
solr_query['q'] = query
solr_query['dictionary'] = collection['suggest']['dictionary']
try:
response = SolrApi(SOLR_URL.get(),
request.user).suggest(collection['name'],
solr_query)
result['response'] = response
result['status'] = 0
except Exception as e:
result['message'] = force_unicode(e)
return JsonResponse(result)
def search(request):
response = {}
collection = json.loads(request.POST.get('collection', '{}'))
query = json.loads(request.POST.get('query', '{}'))
query['download'] = 'download' in request.POST
if collection:
try:
response = SolrApi(SOLR_URL.get(),
request.user).query(collection, query)
response = augment_solr_response(response, collection, query)
except RestException, e:
try:
message = json.loads(e.message)
response['error'] = message['error'].get(
'msg', message['error']['trace'])
except Exception, e2:
LOG.exception('failed to extract json message: %s' %
force_unicode(e2))
LOG.exception('failed to parse json response: %s' %
force_unicode(e))
response['error'] = force_unicode(e)
def get_document(request):
result = {'status': -1, 'message': 'Error'}
try:
collection = json.loads(request.POST.get('collection', '{}'))
doc_id = request.POST.get('id')
if doc_id:
result['doc'] = SolrApi(SOLR_URL.get(),
request.user).get(collection['name'],
doc_id)
if result['doc']['doc']:
result['status'] = 0
result['message'] = ''
else:
result['status'] = 1
result['message'] = _('No document was returned by Solr.')
else:
result['message'] = _('This document does not have any index id.')
result['status'] = 1
except Exception, e:
result['message'] = unicode(str(e), "utf8")
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if core:
raise PopupException(_('Cannot remove Solr cores.'))
if api.remove_collection(name):
# Delete instance directory.
solrctl_path = get_solrctl_path()
process = subprocess.Popen([solrctl_path, "instancedir", "--delete", name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
if process.wait() != 0:
LOG.error("Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate())
raise PopupException(_('Could not create instance directory. Check error logs for more info.'))
else:
raise PopupException(_('Could not remove collection. Check error logs for more info.'))
def update_document(request):
result = {'status': -1, 'message': 'Error'}
if not can_edit_index(request.user):
result['message'] = _('Permission to edit the document denied')
return JsonResponse(result)
try:
collection = json.loads(request.POST.get('collection', '{}'))
document = json.loads(request.POST.get('document', '{}'))
doc_id = request.POST.get('id')
if document['hasChanged']:
edits = {
"id": doc_id,
}
version = None # If there is a version, use it to avoid potential concurrent update conflicts
for field in document['details']:
if field['hasChanged']:
edits[field['key']] = {"set": field['value']}
if field['key'] == '_version_':
version = field['value']
if SolrApi(SOLR_URL.get(),
request.user).update(collection['name'],
json.dumps([edits]),
content_type='json',
version=version):
result['status'] = 0
result['message'] = _('Document successfully updated.')
else:
result['status'] = 0
result['message'] = _('Document has no modifications to change.')
except Exception, e:
result['message'] = force_unicode(e)
def _create_facet(collection, user, facet_id, facet_label, facet_field,
widget_type):
properties = {
'sort': 'desc',
'canRange': False,
'stacked': False,
'limit': 10,
'mincount': 0,
'isDate': False,
'andUp': False, # Not used yet
}
solr_api = SolrApi(SOLR_URL.get(), user)
range_properties = _new_range_facet(solr_api, collection, facet_field,
widget_type)
if range_properties:
facet_type = 'range'
properties.update(range_properties)
elif widget_type == 'hit-widget':
facet_type = 'query'
else:
facet_type = 'field'
if widget_type == 'map-widget':
properties['scope'] = 'world'
properties['mincount'] = 1
properties['limit'] = 100
return {
'id': facet_id,
'label': facet_label,
'field': facet_field,
'type': facet_type,
'widgetType': widget_type,
'properties': properties
}
def create_collection(self,
name,
fields,
unique_key_field='id',
df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# solrcloud mode
# Need to remove path afterwards
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(
_('Error in copying Solr configurations.'), detail=e)
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
def _fetch_collections(request):
from search.conf import SOLR_URL
path = request.GET['path']
item = None
name = None
if path:
item = path
if '/' in path:
item, name = path.split('/')
api = SolrApi(SOLR_URL.get(), request.user)
if not item:
return {"databases": ["collections", "configs"]}
elif item and name:
return {"authorizable_link": "/indexer/#edit/%s" % name, "extended_columns": [], "columns": [], "partition_keys": []}
elif item == 'collections':
return {"tables_meta": [{"comment": None, "type": "Table", "name": col} for col in api.collections2()]}
elif item == 'configs':
return {"tables_meta": [{"comment": None, "type": "Table", "name": conf} for conf in api.configs()]}
else:
raise PopupException(_('Authorizable %s could not be retrieved') % path)
def search(request):
response = {}
collection = json.loads(request.POST.get('collection', '{}'))
query = json.loads(request.POST.get('query', '{}'))
query['download'] = 'download' in request.POST
# todo: remove the selected histo facet if multiq
if collection['id']:
hue_collection = Collection.objects.get(id=collection['id']) # TODO perms
if collection:
try:
response = SolrApi(SOLR_URL.get(), request.user).query(collection, query)
response = augment_solr_response(response, collection, query)
except RestException, e:
try:
response['error'] = json.loads(e.message)['error']['msg']
except:
response['error'] = force_unicode(str(e))
except Exception, e:
raise PopupException(e, title=_('Error while accessing Solr'))
response['error'] = force_unicode(str(e))
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field,
df):
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(
_("Instance directory %s already exists! Please remove it from the file system."
) % instancedir)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df,
False)
try:
shutil.move(solr_config_path, instancedir)
finally:
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def create_collection(self,
name,
fields,
unique_key_field='id',
df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# solrcloud mode
# Need to remove path afterwards
tmp_path, solr_config_path = utils.copy_configs(
fields, unique_key_field, df, True)
# Create instance directory.
solrctl_path = get_solrctl_path()
process = subprocess.Popen([
solrctl_path, "instancedir", "--create", name, solr_config_path
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_ZK_ENSEMBLE':
conf.SOLR_ZK_ENSEMBLE.get()
})
status = process.wait()
# Don't want directories laying around
shutil.rmtree(tmp_path)
if status != 0:
LOG.error(
"Could not create instance directory.\nOutput: %s\nError: %s"
% process.communicate())
raise PopupException(
_('Could not create instance directory. '
'Check if solr_zk_ensemble and solrctl_path are correct in Hue config [indexer].'
))
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
process = subprocess.Popen(
[solrctl_path, "instancedir", "--delete", name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()})
if process.wait() != 0:
LOG.error(
"Cloud not delete collection.\nOutput: %s\nError: %s" %
process.communicate())
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
else:
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(conf.CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(
_("Instance directory %s already exists! Please remove it from the file system."
) % instancedir)
tmp_path, solr_config_path = utils.copy_configs(
fields, unique_key_field, df, False)
shutil.move(solr_config_path, instancedir)
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def __init__(self, user):
self.user = user
self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
def __init__(self, user, api=None): self.user = user self.api = api if api is not None else SolrApi(user=self.user)
def is_core(self, core_name): solr_cores = SolrApi(SOLR_URL.get(), self.user).cores() return core_name in solr_cores
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type):
properties = {
'sort': 'desc',
'canRange': False,
'stacked': False,
'limit': 10,
'mincount': 0,
'isDate': False,
'aggregate': {'function': 'unique', 'ops': [], 'percentiles': [{'value': 50}]}
}
if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'):
facet_type = 'pivot'
elif widget_type == 'gradient-map-widget':
facet_type = 'nested'
properties['facets'] = []
properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 10, 'aggregate': 'count'}
properties['scope'] = 'world'
properties['limit'] = 100
else:
solr_api = SolrApi(SOLR_URL.get(), user)
range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type)
if range_properties:
facet_type = 'range'
properties.update(range_properties)
properties['initial_gap'] = properties['gap']
properties['initial_start'] = properties['start']
properties['initial_end'] = properties['end']
else:
facet_type = 'field'
if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget', 'tree2-widget', 'text-facet-widget', 'hit-widget'):
if widget_type == 'text-facet-widget':
properties['type'] = facet_type
if widget_type == 'hit-widget':
facet_type = 'function'
else:
facet_type = 'nested'
properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 10, 'aggregate': {'function': 'unique', 'ops': [], 'percentiles': [{'value': 50}]}}
properties['facets'] = []
properties['domain'] = {'blockParent': [], 'blockChildren': []}
if widget_type == 'pie2-widget':
properties['scope'] = 'stack'
properties['timelineChartType'] = 'bar'
elif widget_type == 'tree2-widget':
properties['scope'] = 'tree'
properties['facets_form']['limit'] = 5
properties['isOldPivot'] = True
else:
properties['scope'] = 'stack'
properties['timelineChartType'] = 'bar'
if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'):
properties['mincount'] = 1
properties['facets'] = []
properties['stacked'] = True
properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5}
if widget_type == 'map-widget':
properties['scope'] = 'world'
properties['limit'] = 100
else:
properties['scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree'
return {
'id': facet_id,
'label': facet_label,
'field': facet_field,
'type': facet_type,
'widgetType': widget_type,
'properties': properties
}
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
else:
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
shutil.move(solr_config_path, instancedir)
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if core:
raise PopupException(_('Cannot remove Solr cores.'))
if api.remove_collection(name):
# Delete instance directory.
def is_collection(self, collection_name): solr_collections = SolrApi(SOLR_URL.get(), self.user).collections() return collection_name in solr_collections
def test_is_solr_cloud_mode(self):
raise SkipTest # collections() no longer work
SolrApi(SOLR_URL.get(), self.user).collections()
def get_solr_collection(self): return SolrApi(SOLR_URL.get(), self.user).collections()
def __init__(self, user, api=None):
self.user = user
self.api = api if api is not None else SolrApi(
SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
def __init__(self, user, cluster):
DashboardApi.__init__(self, user, cluster)
self.api = SolrApi(SOLR_URL.get(), self.user)
def fields_data(self, user):
schema_fields = SolrApi(SOLR_URL.get(), user).fields(self.name)
schema_fields = schema_fields['schema']['fields']
return sorted([self._make_field(field, attributes) for field, attributes in schema_fields.iteritems()])
