def delete_collection(self, name, core): """ Delete solr collection/core and instance dir """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) client = SolrClient(self.user) if core: raise PopupException(_('Cannot remove Solr cores.')) if api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) with ZookeeperClient(hosts=client.get_zookeeper_host(), read_only=False) as zc: zc.delete_path(root_node) except Exception as e: # Re-create collection so that we don't have an orphan config api.add_collection(name) raise PopupException( _('Error in deleting Solr configurations.'), detail=e) else: raise PopupException( _('Could not remove collection. Check error logs for more info.' ))
def test_get_ensemble_upstream_solr(self): client = SolrClient(self.user, api=MockSolrUpstreamCloudApi()) client._reset_properties() assert_true(client.is_solr_cloud_mode()) assert_true(client.is_solr_six_or_more()) assert_false(client.is_solr_with_hdfs()) assert_equal('localhost:9983', client.get_zookeeper_host())
def test_get_ensemble_cdh_solr(self): client = SolrClient(self.user, api=MockSolrCdhCloudHdfsApi()) client._reset_properties() assert_true(client.is_solr_cloud_mode()) assert_false(client.is_solr_six_or_more()) assert_true(client.is_solr_with_hdfs()) assert_equal('hue.com:2181/solr', client.get_zookeeper_host())
def run_morphline(self, request, collection_name, morphline, input_path, query=None, start_time=None, lib_path=None): workspace_path = self._upload_workspace(morphline) task = make_notebook( name=_('Indexing into %s') % collection_name, editor_type='notebook', on_success_url=reverse('search:browse', kwargs={'name': collection_name}), pub_sub_url='assist.collections.refresh', is_task=True, is_notebook=True, last_executed=start_time ) if query: q = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=query)) notebook_data = q.get_data() snippet = notebook_data['snippets'][0] api = get_api(request, snippet) destination = '__hue_%s' % notebook_data['uuid'][:4] location = '/user/%s/__hue-%s' % (request.user, notebook_data['uuid'][:4]) sql, _success_url = api.export_data_as_table(notebook_data, snippet, destination, is_temporary=True, location=location) input_path = '${nameNode}%s' % location task.add_hive_snippet(snippet['database'], sql) client = SolrClient(self.user) extra_args = ['-Dmapreduce.job.user.classpath.first=true'] if client.is_solr_six_or_more() else [] task.add_java_snippet( clazz='org.apache.solr.hadoop.MapReduceIndexerTool', app_jar=lib_path if lib_path is not None else CONFIG_INDEXER_LIBS_PATH.get(), arguments=extra_args + [ u'--morphline-file', u'morphline.conf', u'--output-dir', u'${nameNode}/user/%s/indexer' % self.username, u'--log4j', u'log4j.properties', u'--go-live', u'--zk-host', client.get_zookeeper_host(), u'--collection', collection_name, input_path, ], files=[ {u'path': u'%s/log4j.properties' % workspace_path, u'type': u'file'}, {u'path': u'%s/morphline.conf' % workspace_path, u'type': u'file'} ] ) return task.execute(request, batch=True)
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df): client = SolrClient(self.user) with ZookeeperClient(hosts=client.get_zookeeper_host(), read_only=False) as zc: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) tmp_path, solr_config_path = copy_configs( fields=fields, unique_key_field=unique_key_field, df=df, solr_cloud_mode=client.is_solr_cloud_mode(), is_solr_six_or_more=client.is_solr_six_or_more(), is_solr_hdfs_mode=client.is_solr_with_hdfs()) try: config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException( _('Error in copying Solr configurations: %s') % e) finally: # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException( _('Error in deleting Solr configurations.'), detail=e) raise PopupException( _('Could not create collection. Check error logs for more info.' ))