def _get_server_properties():
global _api_cache
if not _api_cache: # If we need to refresh the list or if previously no servers were up
_api_cache_lock.acquire()
try:
if not _api_cache:
servers = []
client = ZookeeperClient(hosts=get_sentry_server_ha_zookeeper_quorum())
sentry_servers = client.get_children_data(namespace=get_sentry_server_ha_zookeeper_namespace())
for data in sentry_servers:
server = json.loads(data.decode("utf-8"))
servers.append(
{
"hostname": server["address"],
"port": server["sslPort"] if server["sslPort"] else server["port"],
}
)
_api_cache = servers
except Exception, e:
raise PopupException(_("Error in retrieving Sentry server properties from Zookeeper."), detail=e)
finally:
def test_copy_and_delete_path(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
# Test copy_path
client.copy_path(root_node, TestWithZooKeeper.local_directory)
client.zk.start()
try:
assert_true(client.zk.exists('%s' % root_node))
assert_true(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)))
assert_true(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)))
contents, stats = client.zk.get('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))
assert_equal(contents, TestWithZooKeeper.file_contents)
finally:
client.zk.stop()
# Test delete_path
client.delete_path(root_node)
client.zk.start()
try:
assert_equal(client.zk.exists('%s' % root_node), None)
assert_equal(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None)
assert_equal(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None)
finally:
client.zk.stop()
def test_copy_and_delete_path(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_copy_and_delete_path')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
# Delete the root_node first just in case it wasn't cleaned up in previous run
client.zk.start()
client.zk.delete(root_node, recursive=True)
client.zk.stop()
# Test copy_path
client.copy_path(root_node, TestWithZooKeeper.local_directory)
client.zk.start()
assert_true(client.zk.exists('%s' % root_node))
assert_true(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)))
assert_true(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)))
contents, stats = client.zk.get('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename))
assert_equal(contents, TestWithZooKeeper.file_contents)
client.zk.stop()
# Test delete_path
client.delete_path(root_node)
client.zk.start()
assert_equal(client.zk.exists('%s' % root_node), None)
assert_equal(client.zk.exists('%s/%s' % (root_node, TestWithZooKeeper.subdir_name)), None)
assert_equal(client.zk.exists('%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name, TestWithZooKeeper.filename)), None)
def delete_collection(self, name):
if self.api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
self.api.add_collection(name)
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def test_get_children_data(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
client.zk.start()
try:
client.zk.create(root_node, value='test_path_exists', makepath=True)
finally:
client.zk.stop()
db = client.get_children_data(namespace=TestWithZooKeeper.namespace)
assert_true(len(db) > 0)
def delete_collection(self, name):
if self.api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
zc = ZookeeperClient(hosts=get_solr_ensemble(),
read_only=False)
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
self.api.add_collection(name)
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
def test_get_children_data(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
client.zk.start()
try:
client.zk.create(root_node,
value='test_path_exists',
makepath=True)
finally:
client.zk.stop()
db = client.get_children_data(namespace=TestWithZooKeeper.namespace)
assert_true(len(db) > 0)
def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# solrcloud mode
# Need to remove path afterwards
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(_('Error in copying Solr configurations.'), detail=e)
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def test_path_exists(self):
try:
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
# Delete the root_node first just in case it wasn't cleaned up in previous run
client.zk.start()
client.zk.create(root_node, value='test_path_exists')
client.zk.stop()
assert_true(client.path_exists(namespace=root_node))
assert_false(client.path_exists(namespace='bogus_path'))
finally:
client.delete_path(root_node)
def test_path_exists(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
client.zk.start()
try:
client.zk.create(root_node, value='test_path_exists', makepath=True)
try:
assert_true(client.path_exists(namespace=root_node))
assert_false(client.path_exists(namespace='bogus_path'))
finally:
client.delete_path(root_node)
finally:
client.zk.stop()
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
client = SolrClient(self.user)
if core:
raise PopupException(_('Cannot remove Solr cores.'))
if api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
with ZookeeperClient(hosts=client.get_zookeeper_host(),
read_only=False) as zc:
zc.delete_path(root_node)
except Exception as e:
# Re-create collection so that we don't have an orphan config
api.add_collection(name)
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
else:
raise PopupException(
_('Could not remove collection. Check error logs for more info.'
))
def delete_index(self, name, keep_config=True):
if not self.is_solr_cloud_mode():
raise PopupException(_('Cannot remove non-Solr cloud cores.'))
result = self.api.delete_collection(name)
if result['status'] == 0:
# Delete instance directory.
if not keep_config:
if self.is_solr_six_or_more():
return self.api.delete_config(name)
else:
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
with ZookeeperClient(hosts=self.get_zookeeper_host(),
read_only=False) as zc:
zc.delete_path(root_node)
except Exception as e:
# Re-create collection so that we don't have an orphan config
self.api.add_collection(name)
raise PopupException(
_('Error in deleting Solr configurations.'),
detail=e)
else:
if not 'Cannot unload non-existent core' in json.dumps(result):
raise PopupException(
_('Could not remove collection: %(message)s') % result)
def _create_cloud_config(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=self.get_zookeeper_host(),
read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(
fields=fields,
unique_key_field=unique_key_field,
df=df,
solr_cloud_mode=True,
is_solr_six_or_more=self.is_solr_six_or_more(),
is_solr_hdfs_mode=self.is_solr_with_hdfs(),
is_sentry_protected=self.is_sentry_protected())
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
if not zc.path_exists(root_node):
zc.copy_path(root_node, config_root_path)
else:
LOG.warn('Config %s already existing.' % name)
except Exception as e:
if zc.path_exists(root_node):
zc.delete_path(root_node)
raise PopupException(_('Could not create index: %s') % e)
finally:
shutil.rmtree(tmp_path)
def _create_solr_cloud_collection(self, name, fields, unique_key_field,
df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(
_('Error in copying Solr configurations.'), detail=e)
finally:
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def _get_server_properties():
global _api_cache
if not _api_cache: # If we need to refresh the list or if previously no servers were up
_api_cache_lock.acquire()
try:
if not _api_cache:
servers = []
with ZookeeperClient(
hosts=get_sentry_server_ha_zookeeper_quorum(
)) as client:
sentry_servers = client.get_children_data(
namespace=get_sentry_server_ha_zookeeper_namespace())
for data in sentry_servers:
server = json.loads(data.decode("utf-8"))
servers.append({
'hostname':
server['address'],
'port':
server['sslPort']
if server['sslPort'] else server['port']
})
_api_cache = servers
except Exception, e:
raise PopupException(_(
'Error in retrieving Sentry server properties from Zookeeper.'
),
detail=e)
finally:
def test_copy_and_delete_path(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace,
'test_copy_and_delete_path')
with ZookeeperClient(hosts=zkensemble(), read_only=False) as client:
# Test copy_path
client.copy_path(root_node, TestWithZooKeeper.local_directory)
assert_true(client.zk.exists('%s' % root_node))
assert_true(
client.zk.exists('%s/%s' %
(root_node, TestWithZooKeeper.subdir_name)))
assert_true(
client.zk.exists('%s/%s/%s' %
(root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename)))
contents, stats = client.zk.get(
'%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename))
assert_equal(contents, TestWithZooKeeper.file_contents)
# Test delete_path
client.delete_path(root_node)
assert_equal(client.zk.exists('%s' % root_node), None)
assert_equal(
client.zk.exists('%s/%s' %
(root_node, TestWithZooKeeper.subdir_name)),
None)
assert_equal(
client.zk.exists('%s/%s/%s' %
(root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename)), None)
def _create_solr_cloud_index(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if is_enabled():
with open(
os.path.join(config_root_path,
'solrconfig.xml.secure')) as f:
zc.set(
os.path.join(root_node, 'conf', 'solrconfig.xml'),
f.read())
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_(
'Could not create index. Check error logs for more info.'),
detail=e)
finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# Need to remove path afterwards
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(_('Error in copying Solr configurations.'), detail=e)
# Don't want directories laying around
shutil.rmtree(tmp_path)
if not self.api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def create_index(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
zc = ZookeeperClient(hosts=get_solr_ensemble(),
read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_(
'Could not create index. Check error logs for more info.'),
detail=e)
finally:
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if core:
raise PopupException(_('Cannot remove Solr cores.'))
if api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
api.add_collection(name)
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def teardown(self):
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
# Delete the root_node first just in case it wasn't cleaned up in previous run
client.zk.start()
try:
if client.zk.exists(self.namespace):
client.zk.delete(self.namespace, recursive=True)
finally:
client.zk.stop()
def delete_index(self, name):
"""
Delete solr collection/core and instance dir
"""
# TODO: Implement deletion of local Solr cores
if not self.is_solr_cloud_mode():
raise PopupException(_('Cannot remove non-Solr cloud cores.'))
if self.api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
self.api.add_collection(name)
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def delete_index(self, name):
"""
Delete solr collection/core and instance dir
"""
# TODO: Implement deletion of local Solr cores
if not self.is_solr_cloud_mode():
raise PopupException(_('Cannot remove non-Solr cloud cores.'))
if self.api.remove_collection(name):
# Delete instance directory.
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
zc = ZookeeperClient(hosts=get_solr_ensemble(),
read_only=False)
zc.delete_path(root_node)
except Exception, e:
# Re-create collection so that we don't have an orphan config
self.api.add_collection(name)
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
def test_path_exists(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace, 'test_path_exists')
with ZookeeperClient(hosts=zkensemble(), read_only=False) as client:
client.zk.create(root_node, value='test_path_exists', makepath=True)
try:
assert_true(client.path_exists(namespace=root_node))
assert_false(client.path_exists(namespace='bogus_path'))
finally:
client.delete_path(root_node)
def test_copy_and_delete_path(self):
root_node = '%s/%s' % (TestWithZooKeeper.namespace,
'test_copy_and_delete_path')
client = ZookeeperClient(hosts=zkensemble(), read_only=False)
# Delete the root_node first just in case it wasn't cleaned up in previous run
client.zk.start()
client.zk.delete(root_node, recursive=True)
client.zk.stop()
# Test copy_path
client.copy_path(root_node, TestWithZooKeeper.local_directory)
client.zk.start()
assert_true(client.zk.exists('%s' % root_node))
assert_true(
client.zk.exists('%s/%s' %
(root_node, TestWithZooKeeper.subdir_name)))
assert_true(
client.zk.exists('%s/%s/%s' %
(root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename)))
contents, stats = client.zk.get(
'%s/%s/%s' % (root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename))
assert_equal(contents, TestWithZooKeeper.file_contents)
client.zk.stop()
# Test delete_path
client.delete_path(root_node)
client.zk.start()
assert_equal(client.zk.exists('%s' % root_node), None)
assert_equal(
client.zk.exists('%s/%s' %
(root_node, TestWithZooKeeper.subdir_name)), None)
assert_equal(
client.zk.exists('%s/%s/%s' %
(root_node, TestWithZooKeeper.subdir_name,
TestWithZooKeeper.filename)), None)
def create_index(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
try:
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
finally:
def test_get_children_data(self): client = ZookeeperClient(hosts=zkensemble()) db = client.get_children_data(namespace='') assert_true(len(db) > 0)
def test_get_children_data(self):
client = ZookeeperClient(hosts=zkensemble())
db = client.get_children_data(namespace='')
assert_true(len(db) > 0)
def teardown(self):
with ZookeeperClient(hosts=zkensemble(), read_only=False) as client:
if client.zk.exists(self.namespace):
client.zk.delete(self.namespace, recursive=True)
