def _get_server_properties(): global _api_cache if not _api_cache: # If we need to refresh the list or if previously no servers were up _api_cache_lock.acquire() try: if not _api_cache: servers = [] client = ZookeeperClient(hosts=get_sentry_server_ha_zookeeper_quorum()) sentry_servers = client.get_children_data(namespace=get_sentry_server_ha_zookeeper_namespace()) for data in sentry_servers: server = json.loads(data.decode("utf-8")) servers.append( { "hostname": server["address"], "port": server["sslPort"] if server["sslPort"] else server["port"], } ) _api_cache = servers except Exception, e: raise PopupException(_("Error in retrieving Sentry server properties from Zookeeper."), detail=e) finally:
def test_copy_and_delete_path(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path') client = ZookeeperClient(hosts=zkensemble(), read_only=False) # Test copy_path client.copy_path(root_node, TestWithZooKeeper.local_directory) client.zk.start() try: assert_true(client.zk.exists('%s' % root_node)) assert_true(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name))) assert_true(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))) contents, stats = client.zk.get('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)) assert_equal(contents, TestWithZooKeeper.file_contents) finally: client.zk.stop() # Test delete_path client.delete_path(root_node) client.zk.start() try: assert_equal(client.zk.exists('%s' % root_node), None) assert_equal(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None) assert_equal(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None) finally: client.zk.stop()
def test_copy_and_delete_path(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path') client = ZookeeperClient(hosts=zkensemble(), read_only=False) # Delete the root_node first just in case it wasn't cleaned up in previous run client.zk.start() client.zk.delete(root_node, recursive=True) client.zk.stop() # Test copy_path client.copy_path(root_node, TestWithZooKeeper.local_directory) client.zk.start() assert_true(client.zk.exists('%s' % root_node)) assert_true(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name))) assert_true(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))) contents, stats = client.zk.get('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)) assert_equal(contents, TestWithZooKeeper.file_contents) client.zk.stop() # Test delete_path client.delete_path(root_node) client.zk.start() assert_equal(client.zk.exists('%s' % root_node), None) assert_equal(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None) assert_equal(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None)
def delete_collection(self, name): if self.api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) zc.delete_path(root_node) except Exception, e: # Re-create collection so that we don't have an orphan config self.api.add_collection(name) raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def test_get_children_data(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists') client = ZookeeperClient(hosts=zkensemble(), read_only=False) client.zk.start() try: client.zk.create(root_node, value='test_path_exists', makepath=True) finally: client.zk.stop() db = client.get_children_data(namespace=TestWithZooKeeper.namespace) assert_true(len(db) > 0)
def delete_collection(self, name): if self.api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) zc.delete_path(root_node) except Exception, e: # Re-create collection so that we don't have an orphan config self.api.add_collection(name) raise PopupException( _('Error in deleting Solr configurations.'), detail=e)
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def test_path_exists(self): try: root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists') client = ZookeeperClient(hosts=zkensemble(), read_only=False) # Delete the root_node first just in case it wasn't cleaned up in previous run client.zk.start() client.zk.create(root_node, value='test_path_exists') client.zk.stop() assert_true(client.path_exists(namespace=root_node)) assert_false(client.path_exists(namespace='bogus_path')) finally: client.delete_path(root_node)
def test_path_exists(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists') client = ZookeeperClient(hosts=zkensemble(), read_only=False) client.zk.start() try: client.zk.create(root_node, value='test_path_exists', makepath=True) try: assert_true(client.path_exists(namespace=root_node)) assert_false(client.path_exists(namespace='bogus_path')) finally: client.delete_path(root_node) finally: client.zk.stop()
def delete_collection(self, name, core): """ Delete solr collection/core and instance dir """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) client = SolrClient(self.user) if core: raise PopupException(_('Cannot remove Solr cores.')) if api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) with ZookeeperClient(hosts=client.get_zookeeper_host(), read_only=False) as zc: zc.delete_path(root_node) except Exception as e: # Re-create collection so that we don't have an orphan config api.add_collection(name) raise PopupException( _('Error in deleting Solr configurations.'), detail=e) else: raise PopupException( _('Could not remove collection. Check error logs for more info.' ))
def delete_index(self, name, keep_config=True): if not self.is_solr_cloud_mode(): raise PopupException(_('Cannot remove non-Solr cloud cores.')) result = self.api.delete_collection(name) if result['status'] == 0: # Delete instance directory. if not keep_config: if self.is_solr_six_or_more(): return self.api.delete_config(name) else: try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) with ZookeeperClient(hosts=self.get_zookeeper_host(), read_only=False) as zc: zc.delete_path(root_node) except Exception as e: # Re-create collection so that we don't have an orphan config self.api.add_collection(name) raise PopupException( _('Error in deleting Solr configurations.'), detail=e) else: if not 'Cannot unload non-existent core' in json.dumps(result): raise PopupException( _('Could not remove collection: %(message)s') % result)
def _create_cloud_config(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=self.get_zookeeper_host(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs( fields=fields, unique_key_field=unique_key_field, df=df, solr_cloud_mode=True, is_solr_six_or_more=self.is_solr_six_or_more(), is_solr_hdfs_mode=self.is_solr_with_hdfs(), is_sentry_protected=self.is_sentry_protected()) try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') if not zc.path_exists(root_node): zc.copy_path(root_node, config_root_path) else: LOG.warn('Config %s already existing.' % name) except Exception as e: if zc.path_exists(root_node): zc.delete_path(root_node) raise PopupException(_('Could not create index: %s') % e) finally: shutil.rmtree(tmp_path)
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException( _('Error in copying Solr configurations.'), detail=e) finally: # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException( _('Error in deleting Solr configurations.'), detail=e) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
def _get_server_properties(): global _api_cache if not _api_cache: # If we need to refresh the list or if previously no servers were up _api_cache_lock.acquire() try: if not _api_cache: servers = [] with ZookeeperClient( hosts=get_sentry_server_ha_zookeeper_quorum( )) as client: sentry_servers = client.get_children_data( namespace=get_sentry_server_ha_zookeeper_namespace()) for data in sentry_servers: server = json.loads(data.decode("utf-8")) servers.append({ 'hostname': server['address'], 'port': server['sslPort'] if server['sslPort'] else server['port'] }) _api_cache = servers except Exception, e: raise PopupException(_( 'Error in retrieving Sentry server properties from Zookeeper.' ), detail=e) finally:
def test_copy_and_delete_path(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path') with ZookeeperClient(hosts=zkensemble(), read_only=False) as client: # Test copy_path client.copy_path(root_node, TestWithZooKeeper.local_directory) assert_true(client.zk.exists('%s' % root_node)) assert_true( client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name))) assert_true( client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))) contents, stats = client.zk.get( '%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)) assert_equal(contents, TestWithZooKeeper.file_contents) # Test delete_path client.delete_path(root_node) assert_equal(client.zk.exists('%s' % root_node), None) assert_equal( client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None) assert_equal( client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None)
def _create_solr_cloud_index(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if is_enabled(): with open( os.path.join(config_root_path, 'solrconfig.xml.secure')) as f: zc.set( os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read()) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_( 'Could not create index. Check error logs for more info.'), detail=e) finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # Need to remove path afterwards tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) # Don't want directories laying around shutil.rmtree(tmp_path) if not self.api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def create_index(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_( 'Could not create index. Check error logs for more info.'), detail=e) finally:
def delete_collection(self, name, core): """ Delete solr collection/core and instance dir """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if core: raise PopupException(_('Cannot remove Solr cores.')) if api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) zc.delete_path(root_node) except Exception, e: # Re-create collection so that we don't have an orphan config api.add_collection(name) raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def teardown(self): client = ZookeeperClient(hosts=zkensemble(), read_only=False) # Delete the root_node first just in case it wasn't cleaned up in previous run client.zk.start() try: if client.zk.exists(self.namespace): client.zk.delete(self.namespace, recursive=True) finally: client.zk.stop()
def delete_index(self, name): """ Delete solr collection/core and instance dir """ # TODO: Implement deletion of local Solr cores if not self.is_solr_cloud_mode(): raise PopupException(_('Cannot remove non-Solr cloud cores.')) if self.api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) zc.delete_path(root_node) except Exception, e: # Re-create collection so that we don't have an orphan config self.api.add_collection(name) raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def delete_index(self, name): """ Delete solr collection/core and instance dir """ # TODO: Implement deletion of local Solr cores if not self.is_solr_cloud_mode(): raise PopupException(_('Cannot remove non-Solr cloud cores.')) if self.api.remove_collection(name): # Delete instance directory. try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) zc.delete_path(root_node) except Exception, e: # Re-create collection so that we don't have an orphan config self.api.add_collection(name) raise PopupException( _('Error in deleting Solr configurations.'), detail=e)
def test_path_exists(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists') with ZookeeperClient(hosts=zkensemble(), read_only=False) as client: client.zk.create(root_node, value='test_path_exists', makepath=True) try: assert_true(client.path_exists(namespace=root_node)) assert_false(client.path_exists(namespace='bogus_path')) finally: client.delete_path(root_node)
def test_copy_and_delete_path(self): root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path') client = ZookeeperClient(hosts=zkensemble(), read_only=False) # Delete the root_node first just in case it wasn't cleaned up in previous run client.zk.start() client.zk.delete(root_node, recursive=True) client.zk.stop() # Test copy_path client.copy_path(root_node, TestWithZooKeeper.local_directory) client.zk.start() assert_true(client.zk.exists('%s' % root_node)) assert_true( client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name))) assert_true( client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))) contents, stats = client.zk.get( '%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)) assert_equal(contents, TestWithZooKeeper.file_contents) client.zk.stop() # Test delete_path client.delete_path(root_node) client.zk.start() assert_equal(client.zk.exists('%s' % root_node), None) assert_equal( client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None) assert_equal( client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None)
def create_index(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e) finally:
def test_get_children_data(self): client = ZookeeperClient(hosts=zkensemble()) db = client.get_children_data(namespace='') assert_true(len(db) > 0)
def teardown(self): with ZookeeperClient(hosts=zkensemble(), read_only=False) as client: if client.zk.exists(self.namespace): client.zk.delete(self.namespace, recursive=True)