def admin_collection_template(request, collection_id): hue_collection = Collection.objects.get(id=collection_id) solr_collection = SolrApi(SOLR_URL.get(), request.user).collection_or_core(hue_collection) sample_data = {} if request.method == 'POST': hue_collection.result.update_from_post(request.POST) hue_collection.result.save() return HttpResponse(json.dumps({}), mimetype="application/json") solr_query = {} solr_query['collection'] = hue_collection.name solr_query['q'] = '' solr_query['fq'] = '' solr_query['rows'] = 5 solr_query['start'] = 0 solr_query['facets'] = 0 try: response = SolrApi(SOLR_URL.get(), request.user).query(solr_query, hue_collection) sample_data = json.dumps(response["response"]["docs"]) except PopupException, e: message = e try: message = json.loads(e.message.message)['error']['msg'] # Try to get the core error except: pass request.error(_('No preview available, some facets are invalid: %s') % message) LOG.exception(e)
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, True) # Create instance directory. solrctl_path = get_solrctl_path() process = subprocess.Popen([solrctl_path, "instancedir", "--create", name, solr_config_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) status = process.wait() # Don't want directories laying around shutil.rmtree(tmp_path) if status != 0: LOG.error("Could not create instance directory.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException(_('Could not create instance directory. ' 'Check if solr_zk_ensemble and solrctl_path are correct in Hue config [indexer].')) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. process = subprocess.Popen([solrctl_path, "instancedir", "--delete", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) if process.wait() != 0: LOG.error("Cloud not delete collection.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException(_('Could not create collection. Check error logs for more info.')) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(conf.CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def fields_data(self, user): schema_fields = SolrApi(SOLR_URL.get(), user).fields(self.name) schema_fields = schema_fields['schema']['fields'] dynamic_fields = SolrApi(SOLR_URL.get(), user).fields(self.name, dynamic=True) dynamic_fields = dynamic_fields['fields'] schema_fields.update(dynamic_fields) return sorted([{'name': str(field), 'type': str(attributes.get('type', ''))} for field, attributes in schema_fields.iteritems()])
def get_fields(self, collection_or_core_name): try: field_data = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()).fields(collection_or_core_name) fields = self._format_flags(field_data['schema']['fields']) except: LOG.exception(_('Could not fetch fields for collection %s.') % collection_or_core_name) raise PopupException(_('Could not fetch fields for collection %s. See logs for more info.') % collection_or_core_name) try: uniquekey = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()).uniquekey(collection_or_core_name) except: LOG.exception(_('Could not fetch unique key for collection %s.') % collection_or_core_name) raise PopupException(_('Could not fetch unique key for collection %s. See logs for more info.') % collection_or_core_name) return uniquekey, fields
def get_all_indexes(self, show_all=False): indexes = [] try: indexes = self.get_solr_collections().keys() except: LOG.exception('failed to get indexes') try: indexes += SolrApi(SOLR_URL.get(), self.user).aliases().keys() except: LOG.exception('failed to get index aliases') if show_all or not indexes: return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys() else: return indexes
def get_collections(self): solr_collections = {} solr_aliases = {} solr_cores = {} try: api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if self.is_solr_cloud_mode(): solr_collections = api.collections() for name in solr_collections: solr_collections[name]['isCoreOnly'] = False solr_aliases = api.aliases() for name in solr_aliases: solr_aliases[name] = { 'isCoreOnly': False, 'isAlias': True, 'collections': solr_aliases[name] } solr_cores = api.cores() for name in solr_cores: solr_cores[name]['isCoreOnly'] = True except Exception, e: LOG.warn('No Zookeeper servlet running on Solr server: %s' % e)
def update_data_from_hdfs(self, fs, collection_or_core_name, fields, path, data_type='separated', indexing_strategy='upload', **kwargs): """ Add hdfs path contents to index """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if indexing_strategy == 'upload': stats = fs.stats(path) if stats.size > MAX_UPLOAD_SIZE: raise PopupException(_('File size is too large to handle!')) else: # Get fields for filtering unique_key, fields = self.get_fields(collection_or_core_name) fields = [{'name': field, 'type': fields[field]['type']} for field in fields] fh = fs.open(path) if data_type == 'log': # Transform to JSON then update data = json.dumps([value for value in field_values_from_log(fh, fields)]) content_type = 'json' elif data_type == 'separated': data = json.dumps([value for value in field_values_from_separated_file(fh, kwargs.get('separator', ','), kwargs.get('quote_character', '"'), fields)], indent=2) content_type = 'json' else: raise PopupException(_('Could not update index. Unknown type %s') % data_type) fh.close() if not api.update(collection_or_core_name, data, content_type=content_type): raise PopupException(_('Could not update index. Check error logs for more info.')) else: raise PopupException(_('Could not update index. Indexing strategy %s not supported.') % indexing_strategy)
def zkensemble(): """ Try to guess the value if no values are specified. """ from django.conf import settings if 'zookeeper' in settings.INSTALLED_APPS: try: # Backward compatibility until Hue 4 from zookeeper.conf import CLUSTERS clusters = CLUSTERS.get() if clusters['default'].HOST_PORTS.get() != 'localhost:2181': return '%s' % clusters['default'].HOST_PORTS.get() except: LOG.warn('Could not get zookeeper ensemble from the zookeeper app') if 'search' in settings.INSTALLED_APPS: try: from search.conf import SOLR_URL parsed = urlparse(SOLR_URL.get()) return "%s:2181" % (parsed.hostname or 'localhost') except: LOG.warn('Could not get zookeeper ensemble from the search app') return "localhost:2181"
def update_data_from_hive(self, db, collection_or_core_name, database, table, columns, indexing_strategy='upload'): """ Add hdfs path contents to index """ # Run a custom hive query and post data to collection from beeswax.server import dbms import tablib api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if indexing_strategy == 'upload': table = db.get_table(database, table) hql = "SELECT %s FROM `%s.%s` %s" % (','.join(columns), database, table.name, db._get_browse_limit_clause(table)) query = dbms.hql_query(hql) try: handle = db.execute_and_wait(query) if handle: result = db.fetch(handle, rows=100) db.close(handle) dataset = tablib.Dataset() dataset.append(columns) for row in result.rows(): dataset.append(row) if not api.update(collection_or_core_name, dataset.csv, content_type='csv'): raise PopupException(_('Could not update index. Check error logs for more info.')) else: raise PopupException(_('Could not update index. Could not fetch any data from Hive.')) except Exception, e: raise PopupException(_('Could not update index.'), detail=e)
def _fetch_collections(request): from libsolr.api import SolrApi from search.conf import SOLR_URL path = request.GET['path'] item = None name = None if path: item = path if '/' in path: item, name = path.split('/') api = SolrApi(SOLR_URL.get(), request.user) if not item: return {"databases": ["collections", "configs", "admin"]} elif item and name: return {"authorizable_link": "/indexer/#edit/%s" % name, "extended_columns": [], "columns": [], "partition_keys": []} elif item == 'collections': return {"tables_meta": [{"comment": None, "type": "Table", "name": col} for col in api.collections2()]} elif item == 'configs': return {"tables_meta": [{"comment": None, "type": "Table", "name": conf} for conf in api.configs()]} elif item == 'admin': return {"tables_meta": [{"comment": None, "type": "Table", "name": 'collections'}, {"comment": None, "type": "Table", "name": "cores"}]} else: raise PopupException(_('Authorizable %s could not be retrieved') % path)
def test_query(self): collection = Collection2(user=self.user, name='log_analytics_demo') collection = json.loads(collection.get_json(self.user)) query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0} SolrApi(SOLR_URL.get(), self.user).query(collection['collection'], query)
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) finally: # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def get_terms(request): result = {'status': -1, 'message': 'Error'} try: collection = json.loads(request.POST.get('collection', '{}')) analysis = json.loads(request.POST.get('analysis', '{}')) field = analysis['name'] properties = { 'terms.limit': 25, 'terms.prefix': analysis['terms']['prefix'] # lower # limit # mincount # maxcount } result['terms'] = SolrApi(SOLR_URL.get(), request.user).terms(collection['name'], field, properties) result['terms'] = pairwise2(field, [], result['terms']['terms'][field]) result['status'] = 0 result['message'] = '' except Exception, e: result['message'] = force_unicode(e) if 'not currently supported' in result['message']: result['status'] = 1 result['message'] = _('This field does not support stats')
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type): properties = { 'sort': 'desc', 'canRange': False, 'stacked': False, 'limit': 10, 'mincount': 0, 'isDate': False, 'andUp': False, # Not used yet } solr_api = SolrApi(SOLR_URL.get(), user) range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type) if range_properties: facet_type = 'range' properties.update(range_properties) elif widget_type == 'hit-widget': facet_type = 'query' else: facet_type = 'field' if widget_type == 'map-widget': properties['scope'] = 'world' properties['mincount'] = 1 properties['limit'] = 100 return { 'id': facet_id, 'label': facet_label, 'field': facet_field, 'type': facet_type, 'widgetType': widget_type, 'properties': properties }
def get_terms(request): result = {"status": -1, "message": "Error"} try: collection = json.loads(request.POST.get("collection", "{}")) analysis = json.loads(request.POST.get("analysis", "{}")) field = analysis["name"] properties = { "terms.limit": 25, "terms.prefix": analysis["terms"]["prefix"] # lower # limit # mincount # maxcount } result["terms"] = SolrApi(SOLR_URL.get(), request.user).terms(collection["name"], field, properties) result["terms"] = pairwise2(field, [], result["terms"]["terms"][field]) result["status"] = 0 result["message"] = "" except Exception, e: result["message"] = force_unicode(e) if "not currently supported" in result["message"]: result["status"] = 1 result["message"] = _("This field does not support stats")
def zkensemble(): """ ZooKeeper Ensemble """ from search.conf import SOLR_URL parsed = urlparse(SOLR_URL.get()) return "%s:2181/solr" % (parsed.hostname or 'localhost')
def index(request): hue_collections = Collection.objects.filter(enabled=True) if not hue_collections: if request.user.is_superuser: return admin_collections(request, True) else: return no_collections(request) init_collection = initial_collection(request, hue_collections) search_form = QueryForm(request.GET, initial_collection=init_collection) response = {} error = {} solr_query = {} if search_form.is_valid(): try: collection_id = search_form.cleaned_data['collection'] hue_collection = Collection.objects.get(id=collection_id) solr_query = search_form.solr_query_dict response = SolrApi(SOLR_URL.get(), request.user).query(solr_query, hue_collection) solr_query['total_pages'] = int(math.ceil((float(response['response']['numFound']) / float(solr_query['rows'])))) solr_query['search_time'] = response['responseHeader']['QTime'] except Exception, e: error['title'] = force_unicode(e.title) if hasattr(e, 'title') else '' error['message'] = force_unicode(str(e))
def index(request): hue_collections = Collection.objects.filter(enabled=True) if not hue_collections: if request.user.is_superuser: return admin_collections(request, True) else: return no_collections(request) initial_collection = request.COOKIES.get('hueSearchLastCollection', 0) search_form = QueryForm(request.GET, initial_collection=initial_collection) response = {} error = {} solr_query = {} hue_collection = None if search_form.is_valid(): collection_id = search_form.cleaned_data['collection'] solr_query['q'] = search_form.cleaned_data['query'].encode('utf8') solr_query['fq'] = search_form.cleaned_data['fq'] if search_form.cleaned_data['sort']: solr_query['sort'] = search_form.cleaned_data['sort'] solr_query['rows'] = search_form.cleaned_data['rows'] or 15 solr_query['start'] = search_form.cleaned_data['start'] or 0 solr_query['facets'] = search_form.cleaned_data['facets'] or 1 try: hue_collection = Collection.objects.get(id=collection_id) solr_query['collection'] = hue_collection.name response = SolrApi(SOLR_URL.get(), request.user).query(solr_query, hue_collection) except Exception, e: error['message'] = unicode(str(e), "utf8")
def update_document(request): result = {'status': -1, 'message': 'Error'} if not can_edit_index(request.user): result['message'] = _('Permission to edit the document denied') return JsonResponse(result) try: collection = json.loads(request.POST.get('collection', '{}')) document = json.loads(request.POST.get('document', '{}')) doc_id = request.POST.get('id') if document['hasChanged']: edits = { "id": doc_id, } version = None # If there is a version, use it to avoid potential concurrent update conflicts for field in document['details']: if field['hasChanged']: edits[field['key']] = {"set": field['value']} if field['key'] == '_version_': version = field['value'] if SolrApi(SOLR_URL.get(), request.user).update(collection['name'], json.dumps([edits]), content_type='json', version=version): result['status'] = 0 result['message'] = _('Document successfully updated.') else: result['status'] = 0 result['message'] = _('Document has no modifications to change.') except Exception, e: result['message'] = force_unicode(e)
def get_all_indexes(self, show_all=False): indexes = [] try: indexes = self.get_solr_collection().keys() except: pass try: indexes += SolrApi(SOLR_URL.get(), self.user).aliases().keys() except: pass if show_all or not indexes: return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys() else: return indexes
def get_all_indexes(self): indexes = [] try: indexes = self.get_solr_collection().keys() except: pass return indexes + SolrApi(SOLR_URL.get(), self.user).cores().keys()
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def get_new_cores(self): try: solr_cores = SolrApi(SOLR_URL.get()).cores() for name in Collection.objects.values_list('name', flat=True): solr_cores.pop(name, None) except Exception, e: solr_cores = [] LOG.warn('No Single core setup on Solr server: %s' % e)
def admin_collection_schema(request, collection_id): hue_collection = Collection.objects.get(id=collection_id) solr_schema = SolrApi(SOLR_URL.get(), request.user).schema(hue_collection.name) content = { 'solr_schema': solr_schema.decode('utf-8') } return HttpResponse(json.dumps(content), mimetype="application/json")
def get_new_collections(self): try: solr_collections = SolrApi(SOLR_URL.get()).collections() for name in Collection.objects.values_list('name', flat=True): solr_collections.pop(name, None) except Exception, e: LOG.warn('No Zookeeper servlet running on Solr server: %s' % e) solr_collections = []
def fields_data(self, user, name): api = SolrApi(SOLR_URL.get(), user) try: schema_fields = api.fields(name) schema_fields = schema_fields['schema']['fields'] except Exception, e: LOG.warn('/luke call did not succeed: %s' % e) fields = api.schema_fields(name) schema_fields = Collection2._make_luke_from_schema_fields(fields)
def get_autocomplete(self): autocomplete = {} try: api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) autocomplete['collections'] = api.collections2() autocomplete['configs'] = api.configs() except Exception, e: LOG.warn('No Zookeeper servlet running on Solr server: %s' % e)
def is_solr_cloud_mode(self): api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not hasattr(self, '_solr_cloud_mode'): try: api.collections() setattr(self, '_solr_cloud_mode', True) except: setattr(self, '_solr_cloud_mode', False) return getattr(self, '_solr_cloud_mode')
def is_solr_cloud_mode(self): api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not hasattr(self, '_solr_cloud_mode'): try: api.collections() setattr(self, '_solr_cloud_mode', True) except Exception, e: LOG.info('Non SolrCloud server: %s' % e) setattr(self, '_solr_cloud_mode', False)
def admin_collection_solr_properties(request, collection_id): hue_collection = Collection.objects.get(id=collection_id) solr_collection = SolrApi(SOLR_URL.get(), request.user).collection_or_core(hue_collection) content = render('admin_collection_properties_solr_properties.mako', request, { 'solr_collection': solr_collection, 'hue_collection': hue_collection, }, force_template=True).content return HttpResponse(json.dumps({'content': content}), mimetype="application/json")
def search(request): response = {} collection = json.loads(request.POST.get('collection', '{}')) query = json.loads(request.POST.get('query', '{}')) query['download'] = 'download' in request.POST # todo: remove the selected histo facet if multiq if collection['id']: hue_collection = Collection.objects.get(id=collection['id']) # TODO perms if collection: try: response = SolrApi(SOLR_URL.get(), request.user).query(collection, query) response = augment_solr_response(response, collection, query) except RestException, e: try: response['error'] = json.loads(e.message)['error']['msg'] except: response['error'] = force_unicode(str(e)) except Exception, e: raise PopupException(e, title=_('Error while accessing Solr')) response['error'] = force_unicode(str(e))
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field, df): # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException( _("Instance directory %s already exists! Please remove it from the file system." ) % instancedir) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) try: shutil.move(solr_config_path, instancedir) finally: shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
def admin_collection_properties(request, collection_id): hue_collection = Collection.objects.get(id=collection_id) solr_collection = SolrApi(SOLR_URL.get(), request.user).collection_or_core(hue_collection) if request.method == 'POST': collection_form = CollectionForm(request.POST, instance=hue_collection, user=request.user) if collection_form.is_valid(): # Check for autocomplete in data? searcher = SearchController(request.user) hue_collection = collection_form.save(commit=False) hue_collection.is_core_only = not searcher.is_collection(hue_collection.name) hue_collection.autocomplete = json.loads(request.POST.get('autocomplete')) hue_collection.save() return redirect(reverse('search:admin_collection_properties', kwargs={'collection_id': hue_collection.id})) else: request.error(_('Errors on the form: %s.') % collection_form.errors) else: collection_form = CollectionForm(instance=hue_collection) return render('admin_collection_properties.mako', request, { 'solr_collection': solr_collection, 'hue_collection': hue_collection, 'collection_form': collection_form, 'collection_properties': json.dumps(hue_collection.properties_dict) })
def index_fields_dynamic(request): result = {'status': -1, 'message': 'Error'} try: name = request.POST['name'] hue_collection = Collection(name=name, label=name) dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(hue_collection.name) result['message'] = '' result['fields'] = [ Collection._make_field(name, properties) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['gridlayout_header_fields'] = [ Collection._make_gridlayout_header_field({'name': name}, True) for name, properties in dynamic_fields['fields'].iteritems() if 'dynamicBase' in properties ] result['status'] = 0 except Exception, e: result['message'] = unicode(str(e), "utf8")
def update_document(request): result = {'status': -1, 'message': 'Error'} if not can_edit_index(request.user): result['message'] = _('Permission to edit the document denied') return JsonResponse(result) try: collection = json.loads(request.POST.get('collection', '{}')) document = json.loads(request.POST.get('document', '{}')) doc_id = request.POST.get('id') if document['hasChanged']: edits = { "id": doc_id, } version = None # If there is a version, use it to avoid potential concurrent update conflicts for field in document['details']: if field['hasChanged'] and field['key'] != '_version_': edits[field['key']] = {"set": field['value']} if field['key'] == '_version_': version = field['value'] result['update'] = SolrApi(SOLR_URL.get(), request.user).update(collection['name'], json.dumps([edits]), content_type='json', version=version) result['message'] = _('Document successfully updated.') result['status'] = 0 else: result['status'] = 0 result['message'] = _('Document has no modifications to change.') except RestException, e: try: result['message'] = json.loads(e.message)['error']['msg'] except: LOG.exception('Failed to parse json response') result['message'] = force_unicode(e)
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type): properties = { 'sort': 'desc', 'canRange': False, 'stacked': False, 'limit': 10, 'mincount': 0, 'isDate': False, 'aggregate': 'unique' } if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): facet_type = 'pivot' elif widget_type == 'hit-widget': facet_type = 'function' else: solr_api = SolrApi(SOLR_URL.get(), user) range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type) if range_properties: facet_type = 'range' properties.update(range_properties) properties['initial_gap'] = properties['gap'] properties['initial_start'] = properties['start'] properties['initial_end'] = properties['end'] else: facet_type = 'field' if widget_type == 'bucket-widget': facet_type = 'nested' properties['facets_form'] = { 'field': '', 'mincount': 1, 'limit': 10, 'aggregate': 'count' } properties['facets'] = [] properties['scope'] = 'stack' if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): properties['mincount'] = 1 properties['facets'] = [] properties['stacked'] = True properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5} if widget_type == 'map-widget': properties['scope'] = 'world' properties['limit'] = 100 else: properties[ 'scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree' return { 'id': facet_id, 'label': facet_label, 'field': facet_field, 'type': facet_type, 'widgetType': widget_type, 'properties': properties }
def fields_data(self, user): schema_fields = SolrApi(SOLR_URL.get(), user).fields(self.name) schema_fields = schema_fields['schema']['fields'] return sorted([self._make_field(field, attributes) for field, attributes in schema_fields.iteritems()])
def _envelope_job(request, file_format, destination, start_time=None, lib_path=None): collection_name = destination['name'] indexer = EnvelopeIndexer(request.user, request.fs) lib_path = None # Todo optional input field input_path = None if file_format['inputFormat'] == 'table': db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) input_path = table_metadata.path_location elif file_format['inputFormat'] == 'file': input_path = file_format["path"] properties = {'input_path': input_path, 'format': 'csv'} elif file_format['inputFormat'] == 'stream' and file_format[ 'streamSelection'] == 'flume': pass elif file_format['inputFormat'] in ('stream', 'sfdc'): if file_format['inputFormat'] == 'sfdc': properties = { 'streamSelection': file_format['streamSelection'], 'streamUsername': file_format['streamUsername'], 'streamPassword': file_format['streamPassword'], 'streamToken': file_format['streamToken'], 'streamEndpointUrl': file_format['streamEndpointUrl'], 'streamObject': file_format['streamObject'], } elif file_format['streamSelection'] == 'kafka': manager = ManagerApi() properties = { "brokers": manager.get_kafka_brokers(), "topics": file_format['kafkaSelectedTopics'], "kafkaFieldType": file_format['kafkaFieldType'], "kafkaFieldDelimiter": file_format['kafkaFieldDelimiter'], "kafkaFieldNames": file_format['kafkaFieldNames'], "kafkaFieldTypes": file_format['kafkaFieldTypes'] } if True: properties['window'] = '' else: # For "KafkaSQL" properties['window'] = ''' window { enabled = true milliseconds = 60000 }''' if destination['outputFormat'] == 'table': if destination['isTargetExisting']: # Todo: check if format matches pass else: sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file( file_format, destination).get_str() print sql if destination['tableFormat'] == 'kudu': manager = ManagerApi() properties["output_table"] = "impala::%s" % collection_name properties["kudu_master"] = manager.get_kudu_master() else: properties['output_table'] = collection_name elif destination['outputFormat'] == 'file': properties['path'] = file_format["path"] if file_format['inputFormat'] == 'stream': properties['format'] = 'csv' else: properties['format'] = file_format['tableFormat'] # or csv elif destination['outputFormat'] == 'index': properties['collectionName'] = collection_name properties['connection'] = SOLR_URL.get() # No needed anymore # if destination['isTargetExisting']: # # Todo: check if format matches # pass # else: # client = SolrClient(request.user) # kwargs = {} # _create_solr_collection(request.user, request.fs, client, destination, collection_name, kwargs) if destination['outputFormat'] == 'stream': manager = ManagerApi() properties['brokers'] = manager.get_kafka_brokers() properties['topics'] = file_format['kafkaSelectedTopics'] properties['kafkaFieldDelimiter'] = file_format['kafkaFieldDelimiter'] properties["app_name"] = 'Data Ingest' properties["inputFormat"] = file_format['inputFormat'] properties["ouputFormat"] = destination['ouputFormat'] properties["streamSelection"] = file_format["streamSelection"] envelope = indexer.generate_config(properties) return indexer.run(request, collection_name, envelope, input_path, start_time=start_time, lib_path=lib_path)
def test_is_solr_cloud_mode(self): SolrApi(SOLR_URL.get(), self.user).collections()
# Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException(_('Could not create collection. Check error logs for more info.')) def delete_collection(self, name, core): """ Delete solr collection/core and instance dir """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if core: raise PopupException(_('Cannot remove Solr cores.')) if api.remove_collection(name): # Delete instance directory.
hue_collection = None if search_form.is_valid(): collection_id = search_form.cleaned_data['collection'] solr_query['q'] = search_form.cleaned_data['query'].encode('utf8') solr_query['fq'] = search_form.cleaned_data['fq'] if search_form.cleaned_data['sort']: solr_query['sort'] = search_form.cleaned_data['sort'] solr_query['rows'] = search_form.cleaned_data['rows'] or 15 solr_query['start'] = search_form.cleaned_data['start'] or 0 solr_query['facets'] = search_form.cleaned_data['facets'] or 1 try: hue_collection = Collection.objects.get(id=collection_id) solr_query['collection'] = hue_collection.name response = SolrApi(SOLR_URL.get(), request.user).query(solr_query, hue_collection) except Exception, e: error['message'] = unicode(str(e), "utf8") else: error['message'] = _('There is no collection to search.') if hue_collection is not None: response = augment_solr_response(response, hue_collection.facets.get_data()) if request.GET.get('format') == 'json': return HttpResponse(json.dumps(response), mimetype="application/json") return render( 'search.mako', request, {
def __init__(self, user, cluster): DashboardApi.__init__(self, user, cluster) self.api = SolrApi(SOLR_URL.get(), self.user)
def _envelope_job(request, file_format, destination, start_time=None, lib_path=None): collection_name = destination['name'] indexer = EnvelopeIndexer(request.user, request.fs) lib_path = None # Todo optional input field input_path = None if file_format['inputFormat'] == 'table': db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) input_path = table_metadata.path_location elif file_format['inputFormat'] == 'file': input_path = file_format["path"] properties = {'input_path': input_path, 'format': 'csv'} elif file_format['inputFormat'] == 'stream' and file_format[ 'streamSelection'] == 'flume': pass elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': manager = ManagerApi() properties = { "brokers": manager.get_kafka_brokers(), "topics": file_format['kafkaSelectedTopics'], "kafkaFieldType": file_format['kafkaFieldType'], "kafkaFieldDelimiter": file_format['kafkaFieldDelimiter'], } if file_format.get( 'kafkaSelectedTopics') == 'NavigatorAuditEvents': schema_fields = MorphlineIndexer.get_kept_field_list( file_format['sampleCols']) properties.update({ "kafkaFieldNames": ', '.join([_field['name'] for _field in schema_fields]), "kafkaFieldTypes": ', '.join([_field['type'] for _field in schema_fields]) }) else: properties.update({ "kafkaFieldNames": file_format['kafkaFieldNames'], "kafkaFieldTypes": file_format['kafkaFieldTypes'] }) if True: properties['window'] = '' else: # For "KafkaSQL" properties['window'] = ''' window { enabled = true milliseconds = 60000 }''' elif file_format['inputFormat'] == 'connector': if file_format['streamSelection'] == 'flume': properties = { 'streamSelection': file_format['streamSelection'], 'channelSourceHosts': file_format['channelSourceHosts'], 'channelSourceSelectedHosts': file_format['channelSourceSelectedHosts'], 'channelSourcePath': file_format['channelSourcePath'], } else: # sfdc properties = { 'streamSelection': file_format['streamSelection'], 'streamUsername': file_format['streamUsername'], 'streamPassword': file_format['streamPassword'], 'streamToken': file_format['streamToken'], 'streamEndpointUrl': file_format['streamEndpointUrl'], 'streamObject': file_format['streamObject'], } if destination['outputFormat'] == 'table': if destination['isTargetExisting']: # Todo: check if format matches pass else: destination['importData'] = False # Avoid LOAD DATA if destination['tableFormat'] == 'kudu': properties['kafkaFieldNames'] = properties[ 'kafkaFieldNames'].lower( ) # Kudu names should be all lowercase # Create table if not request.POST.get('show_command'): SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file( file_format, destination).execute(request) if destination['tableFormat'] == 'kudu': manager = ManagerApi() properties["output_table"] = "impala::%s" % collection_name properties["kudu_master"] = manager.get_kudu_master() else: properties['output_table'] = collection_name elif destination['outputFormat'] == 'stream': manager = ManagerApi() properties['brokers'] = manager.get_kafka_brokers() properties['topics'] = file_format['kafkaSelectedTopics'] properties['kafkaFieldDelimiter'] = file_format['kafkaFieldDelimiter'] elif destination['outputFormat'] == 'file': properties['path'] = file_format["path"] if file_format['inputFormat'] == 'stream': properties['format'] = 'csv' else: properties['format'] = file_format['tableFormat'] # or csv elif destination['outputFormat'] == 'index': properties['collectionName'] = collection_name properties['connection'] = SOLR_URL.get() properties["app_name"] = 'Data Ingest' properties["inputFormat"] = file_format['inputFormat'] properties["ouputFormat"] = destination['ouputFormat'] properties["streamSelection"] = file_format["streamSelection"] configs = indexer.generate_config(properties) if request.POST.get('show_command'): return {'status': 0, 'commands': configs['envelope.conf']} else: return indexer.run(request, collection_name, configs, input_path, start_time=start_time, lib_path=lib_path)
class SolrApi(Api): def __init__(self, user, interpreter=None): Api.__init__(self, user, interpreter=interpreter) self.options = interpreter['options'] @query_error_handler def execute(self, notebook, snippet): from search.conf import SOLR_URL api = NativeSolrApi(SOLR_URL.get(), self.user.username) collection = self.options.get('collection') or snippet.get('database') if not collection or collection == 'default': collection = api.collections2()[0] response = api.sql(collection, snippet['statement']) info = response['result-set']['docs'].pop( -1) # EOF, RESPONSE_TIME, EXCEPTION if info.get('EXCEPTION'): raise QueryError(info['EXCEPTION']) headers = [] for row in response['result-set']['docs']: for col in row.keys(): if col not in headers: headers.append(col) data = [[doc.get(col) for col in headers] for doc in response['result-set']['docs']] has_result_set = bool(data) return { 'sync': True, 'has_result_set': has_result_set, 'modified_row_count': 0, 'result': { 'has_more': False, 'data': data if has_result_set else [], 'meta': [{ 'name': col, 'type': '', 'comment': '' } for col in headers] if has_result_set else [], 'type': 'table' }, 'statement_id': 0, 'has_more_statements': False, 'statements_count': 1 } @query_error_handler def check_status(self, notebook, snippet): return {'status': 'available'} @query_error_handler def fetch_result(self, notebook, snippet, rows, start_over): return {'has_more': False, 'data': [], 'meta': [], 'type': 'table'} @query_error_handler def fetch_result_metadata(self): pass @query_error_handler def cancel(self, notebook, snippet): return {'status': 0} @query_error_handler def get_log(self, notebook, snippet, startFrom=None, size=None): return 'No logs' def download(self, notebook, snippet, format): raise PopupException('Downloading is not supported yet') @query_error_handler def close_statement(self, snippet): return {'status': -1} @query_error_handler def autocomplete(self, snippet, database=None, table=None, column=None, nested=None): from search.conf import SOLR_URL api = NativeSolrApi(SOLR_URL.get(), self.user.username) assist = Assist(self, self.user, api) response = {'status': -1} if database is None: response['databases'] = [ self.options.get('collection') or snippet.get('database') or 'default' ] elif table is None: response['tables_meta'] = assist.get_tables(database) else: columns = assist.get_columns(database, table) response['columns'] = [col['name'] for col in columns] response['extended_columns'] = columns response['status'] = 0 return response @query_error_handler def get_sample_data(self, snippet, database=None, table=None, column=None, async=False): from search.conf import SOLR_URL db = NativeSolrApi(SOLR_URL.get(), self.user) assist = Assist(self, self.user, db) response = {'status': -1} if snippet.get('source') == 'sql': sample_data = assist.get_sample_data_sql(database, table, column) else: sample_data = assist.get_sample_data(database, table, column) if sample_data: response['status'] = 0 response['headers'] = sample_data['headers'] response['full_headers'] = sample_data.get('full_headers') response['rows'] = sample_data['rows'] else: response['message'] = _('Failed to get sample data.') return response
def is_core(self, core_name): solr_cores = SolrApi(SOLR_URL.get()).cores() return core_name in solr_cores
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type): properties = { 'sort': 'desc', 'canRange': False, 'stacked': False, 'limit': 10, 'mincount': 0, 'isDate': False, 'aggregate': {'function': 'unique', 'ops': [], 'percentiles': [{'value': 50}]} } if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): facet_type = 'pivot' elif widget_type == 'gradient-map-widget': facet_type = 'nested' properties['facets'] = [] properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 10, 'aggregate': 'count'} properties['scope'] = 'world' properties['limit'] = 100 else: solr_api = SolrApi(SOLR_URL.get(), user) range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type) if range_properties: facet_type = 'range' properties.update(range_properties) properties['initial_gap'] = properties['gap'] properties['initial_start'] = properties['start'] properties['initial_end'] = properties['end'] else: facet_type = 'field' if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget', 'tree2-widget', 'text-facet-widget', 'hit-widget'): if widget_type == 'text-facet-widget': properties['type'] = facet_type if widget_type == 'hit-widget': facet_type = 'function' else: facet_type = 'nested' properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 10, 'aggregate': {'function': 'unique', 'ops': [], 'percentiles': [{'value': 50}]}} properties['facets'] = [] properties['domain'] = {'blockParent': [], 'blockChildren': []} if widget_type == 'pie2-widget': properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' elif widget_type == 'tree2-widget': properties['scope'] = 'tree' properties['facets_form']['limit'] = 5 properties['isOldPivot'] = True else: properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): properties['mincount'] = 1 properties['facets'] = [] properties['stacked'] = True properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5} if widget_type == 'map-widget': properties['scope'] = 'world' properties['limit'] = 100 else: properties['scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree' return { 'id': facet_id, 'label': facet_label, 'field': facet_field, 'type': facet_type, 'widgetType': widget_type, 'properties': properties }
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = utils.copy_configs( fields, unique_key_field, df, True) # Create instance directory. solrctl_path = get_solrctl_path() process = subprocess.Popen([ solrctl_path, "instancedir", "--create", name, solr_config_path ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) status = process.wait() # Don't want directories laying around shutil.rmtree(tmp_path) if status != 0: LOG.error( "Could not create instance directory.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException( _('Could not create instance directory. ' 'Check if solr_zk_ensemble and solrctl_path are correct in Hue config [indexer].' )) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. process = subprocess.Popen( [solrctl_path, "instancedir", "--delete", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()}) if process.wait() != 0: LOG.error( "Cloud not delete collection.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException( _('Could not create collection. Check error logs for more info.' )) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(conf.CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException( _("Instance directory %s already exists! Please remove it from the file system." ) % instancedir) tmp_path, solr_config_path = utils.copy_configs( fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
def __init__(self, user): self.user = user self.api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
def test_is_solr_cloud_mode(self): raise SkipTest # collections() no longer work SolrApi(SOLR_URL.get(), self.user).collections()
def _envelope_job(request, file_format, destination, start_time=None, lib_path=None): collection_name = destination['name'] indexer = EnvelopeIndexer(request.user, request.fs) lib_path = '/tmp/envelope-0.5.0.jar' input_path = None if file_format['inputFormat'] == 'table': db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) input_path = table_metadata.path_location elif file_format['inputFormat'] == 'file': input_path = '${nameNode}%s' % file_format["path"] properties = { 'format': 'json' } elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'sfdc': properties = { 'streamSelection': file_format['streamSelection'], 'streamUsername': file_format['streamUsername'], 'streamPassword': file_format['streamPassword'], 'streamToken': file_format['streamToken'], 'streamEndpointUrl': file_format['streamEndpointUrl'], 'streamObject': file_format['streamObject'], } elif file_format['streamSelection'] == 'kafka': manager = ManagerApi() properties = { "brokers": manager.get_kafka_brokers(), "output_table": "impala::%s" % collection_name, "topics": file_format['kafkaSelectedTopics'], "kafkaFieldType": file_format['kafkaFieldType'], "kafkaFieldDelimiter": file_format['kafkaFieldDelimiter'], "kafkaFieldNames": file_format['kafkaFieldNames'], "kafkaFieldTypes": file_format['kafkaFieldTypes'] } if destination['outputFormat'] == 'table': if destination['isTargetExisting']: # Todo: check if format matches pass else: sql = SQLIndexer(user=request.user, fs=request.fs).create_table_from_a_file(file_format, destination).get_str() print sql if destination['tableFormat'] == 'kudu': manager = ManagerApi() properties["output_table"] = "impala::%s" % collection_name properties["kudu_master"] = manager.get_kudu_master() else: properties['output_table'] = collection_name elif destination['outputFormat'] == 'file': properties['path'] = file_format["path"] properties['format'] = file_format['tableFormat'] # or csv elif destination['outputFormat'] == 'index': properties['collectionName'] = collection_name properties['connection'] = SOLR_URL.get() if destination['isTargetExisting']: # Todo: check if format matches pass else: client = SolrClient(request.user) kwargs = {} _create_solr_collection(request.user, request.fs, client, destination, collection_name, kwargs) properties["app_name"] = 'Data Ingest' properties["inputFormat"] = file_format['inputFormat'] properties["ouputFormat"] = destination['ouputFormat'] properties["streamSelection"] = file_format["streamSelection"] envelope = indexer.generate_config(properties) return indexer.run(request, collection_name, envelope, input_path, start_time=start_time, lib_path=lib_path)
def is_collection(self, collection_name): solr_collections = SolrApi(SOLR_URL.get()).collections() return collection_name in solr_collections
def get_solr_collection(self): return SolrApi(SOLR_URL.get(), self.user).collections()
def admin_collection_schema(request, collection_id): hue_collection = Collection.objects.get(id=collection_id) solr_schema = SolrApi(SOLR_URL.get()).schema(hue_collection.name) content = {'solr_schema': solr_schema.decode('utf-8')} return HttpResponse(json.dumps(content), mimetype="application/json")
def __init__(self, user, api=None): self.user = user self.api = api if api is not None else SolrApi( SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
response['columns'] = [col['name'] for col in columns] response['extended_columns'] = columns response['status'] = 0 return response @query_error_handler def get_sample_data(self, snippet, database=None, table=None, column=None, async=False, operation=None): from search.conf import SOLR_URL db = NativeSolrApi(SOLR_URL.get(), self.user) assist = Assist(self, self.user, db) response = {'status': -1} if snippet.get('source') == 'sql': sample_data = assist.get_sample_data_sql(database, table, column) else: sample_data = assist.get_sample_data(database, table, column) if sample_data: response['status'] = 0 response['headers'] = sample_data['headers'] response['full_headers'] = sample_data.get('full_headers') response['rows'] = sample_data['rows'] else:
def _create_facet(collection, user, facet_id, facet_label, facet_field, widget_type): properties = { 'sort': 'desc', 'canRange': False, 'stacked': False, 'limit': 10, 'mincount': 0, 'isDate': False, 'aggregate': { 'function': 'unique', 'ops': [], 'percentiles': [{ 'value': 50 }] } } if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): facet_type = 'pivot' elif widget_type == 'gradient-map-widget': facet_type = 'nested' properties['facets'] = [] properties['domain'] = {'blockParent': [], 'blockChildren': []} properties['facets_form'] = { 'field': '', 'mincount': 1, 'limit': 10, 'aggregate': { 'function': 'unique', 'ops': [], 'percentiles': [{ 'value': 50 }] } } properties['scope'] = 'world' properties['limit'] = 100 else: solr_api = SolrApi(SOLR_URL.get(), user) range_properties = _new_range_facet(solr_api, collection, facet_field, widget_type) if range_properties: facet_type = 'range' properties.update(range_properties) properties['initial_gap'] = properties['gap'] properties['initial_start'] = properties['start'] properties['initial_end'] = properties['end'] else: facet_type = 'field' if widget_type in ('bucket-widget', 'pie2-widget', 'timeline-widget', 'tree2-widget', 'text-facet-widget', 'hit-widget'): if widget_type == 'text-facet-widget': properties['type'] = facet_type if widget_type == 'hit-widget': facet_type = 'function' else: facet_type = 'nested' properties['facets_form'] = { 'field': '', 'mincount': 1, 'limit': 10, 'aggregate': { 'function': 'unique', 'ops': [], 'percentiles': [{ 'value': 50 }] } } properties['facets'] = [] properties['domain'] = {'blockParent': [], 'blockChildren': []} if widget_type == 'pie2-widget': properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' elif widget_type == 'tree2-widget': properties['scope'] = 'tree' properties['facets_form']['limit'] = 5 properties['isOldPivot'] = True else: properties['scope'] = 'stack' properties['timelineChartType'] = 'bar' if widget_type in ('tree-widget', 'heatmap-widget', 'map-widget'): properties['mincount'] = 1 properties['facets'] = [] properties['stacked'] = True properties['facets_form'] = {'field': '', 'mincount': 1, 'limit': 5} if widget_type == 'map-widget': properties['scope'] = 'world' properties['limit'] = 100 else: properties[ 'scope'] = 'stack' if widget_type == 'heatmap-widget' else 'tree' return { 'id': facet_id, 'label': facet_label, 'field': facet_field, 'type': facet_type, 'widgetType': widget_type, 'properties': properties, # Hue 4+ 'template': { "showFieldList": True, "showGrid": False, "showChart": True, "chartSettings": { 'chartType': 'pie' if widget_type == 'pie2-widget' else ('timeline' if widget_type == 'timeline-widget' else ('gradientmap' if widget_type == 'gradient-map-widget' else 'bars')), 'chartSorting': 'none', 'chartScatterGroup': None, 'chartScatterSize': None, 'chartScope': 'world', 'chartX': None, 'chartYSingle': None, 'chartYMulti': [], 'chartData': [], 'chartMapLabel': None, }, "fieldsAttributes": [], "fieldsAttributesFilter": "", "filteredAttributeFieldsAll": True, "fields": [], "fieldsSelected": [], "leafletmap": { 'latitudeField': None, 'longitudeField': None, 'labelField': None }, # Use own? 'leafletmapOn': False, 'isGridLayout': False, "hasDataForChart": True, "rows": 25, } }