def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# solrcloud mode
# Need to remove path afterwards
tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, True)
# Create instance directory.
solrctl_path = get_solrctl_path()
process = subprocess.Popen([solrctl_path, "instancedir", "--create", name, solr_config_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
status = process.wait()
# Don't want directories laying around
shutil.rmtree(tmp_path)
if status != 0:
LOG.error("Could not create instance directory.\nOutput: %s\nError: %s" % process.communicate())
raise PopupException(_('Could not create instance directory. '
'Check if solr_zk_ensemble and solrctl_path are correct in Hue config [indexer].'))
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
process = subprocess.Popen([solrctl_path, "instancedir", "--delete", name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
if process.wait() != 0:
LOG.error("Cloud not delete collection.\nOutput: %s\nError: %s" % process.communicate())
raise PopupException(_('Could not create collection. Check error logs for more info.'))
else:
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(conf.CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, False)
shutil.move(solr_config_path, instancedir)
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_solr_cloud_collection(self, name, fields, unique_key_field,
df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(
_('Error in copying Solr configurations.'), detail=e)
finally:
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def _create_non_solr_cloud_index(self, name, fields, unique_key_field, df):
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(
_("Instance directory %s already exists! Please remove it from the file system."
) % instancedir)
try:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, False)
try:
shutil.move(solr_config_path, instancedir)
finally:
shutil.rmtree(tmp_path)
if not self.api.create_core(name, instancedir):
raise Exception('Failed to create core: %s' % name)
except Exception as e:
raise PopupException(
_('Could not create index. Check error logs for more info.'),
detail=e)
finally:
shutil.rmtree(instancedir)
def _create_cloud_config(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=self.get_zookeeper_host(),
read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(
fields=fields,
unique_key_field=unique_key_field,
df=df,
solr_cloud_mode=True,
is_solr_six_or_more=self.is_solr_six_or_more(),
is_solr_hdfs_mode=self.is_solr_with_hdfs(),
is_sentry_protected=self.is_sentry_protected())
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
if not zc.path_exists(root_node):
zc.copy_path(root_node, config_root_path)
else:
LOG.warn('Config %s already existing.' % name)
except Exception as e:
if zc.path_exists(root_node):
zc.delete_path(root_node)
raise PopupException(_('Could not create index: %s') % e)
finally:
shutil.rmtree(tmp_path)
def _create_solr_cloud_index(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if is_enabled():
with open(
os.path.join(config_root_path,
'solrconfig.xml.secure')) as f:
zc.set(
os.path.join(root_node, 'conf', 'solrconfig.xml'),
f.read())
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_(
'Could not create index. Check error logs for more info.'),
detail=e)
finally:
def create_index(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
tmp_path, solr_config_path = copy_configs(fields, unique_key_field,
df, True)
try:
zc = ZookeeperClient(hosts=get_solr_ensemble(),
read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_(
'Could not create index. Check error logs for more info.'),
detail=e)
finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# Need to remove path afterwards
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(_('Error in copying Solr configurations.'), detail=e)
# Don't want directories laying around
shutil.rmtree(tmp_path)
if not self.api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
# solrcloud mode
# Need to remove path afterwards
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(_('Error in copying Solr configurations.'), detail=e)
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
try:
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(_('Error in copying Solr configurations.'), detail=e)
finally:
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def create_collection(self,
name,
fields,
unique_key_field='id',
df='text'):
"""
Create solr collection and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
# Need to remove path afterwards
tmp_path, solr_config_path = utils.copy_configs(
fields, unique_key_field, df)
# Create instance directory.
process = subprocess.Popen(
[
conf.SOLRCTL_PATH.get(), "instancedir", "--create", name,
solr_config_path
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_HOME': conf.SOLR_HOME.get(),
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
status = process.wait()
shutil.rmtree(tmp_path)
if status != 0:
LOG.error(
"Cloud not create instance directory.\nOutput stream: %s\nError stream: %s"
% process.communicate())
raise PopupException(
_('Could not create instance directory. Check error logs for more info.'
))
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory.
process = subprocess.Popen(
[conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_HOME': conf.SOLR_HOME.get(),
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
if process.wait() != 0:
LOG.error(
"Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s"
% process.communicate())
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
def _create_solr_cloud_index(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
try:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
finally:
def _create_non_solr_cloud_index(self, name, fields, unique_key_field, df):
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
try:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
try:
shutil.move(solr_config_path, instancedir)
finally:
shutil.rmtree(tmp_path)
if not self.api.create_core(name, instancedir):
raise Exception('Failed to create core: %s' % name)
except Exception, e:
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field, df):
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
try:
shutil.move(solr_config_path, instancedir)
finally:
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_solr_cloud_index(self, name, fields, unique_key_field, df):
with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
try:
root_node = "%s/%s" % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = "%s/%s" % (solr_config_path, "conf")
zc.copy_path(root_node, config_root_path)
if is_enabled():
with open(os.path.join(config_root_path, "solrconfig.xml.secure")) as f:
zc.set(os.path.join(root_node, "conf", "solrconfig.xml"), f.read())
if not self.api.create_collection(name):
raise Exception("Failed to create collection: %s" % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_("Could not create index. Check error logs for more info."), detail=e)
finally:
def create_index(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection or core and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
if self.is_solr_cloud_mode():
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True)
try:
zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False)
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
config_root_path = '%s/%s' % (solr_config_path, 'conf')
zc.copy_path(root_node, config_root_path)
if not self.api.create_collection(name):
raise Exception('Failed to create collection: %s' % name)
except Exception, e:
if zc.path_exists(root_node):
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'):
"""
Create solr collection and instance dir.
Create schema.xml file so that we can set UniqueKey field.
"""
# Need to remove path afterwards
tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df)
# Create instance directory.
process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_HOME': conf.SOLR_HOME.get(),
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
status = process.wait()
shutil.rmtree(tmp_path)
if status != 0:
LOG.error("Cloud not create instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate())
raise PopupException(_('Could not create instance directory. Check error logs for more info.'))
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory.
process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'SOLR_HOME': conf.SOLR_HOME.get(),
'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get()
})
if process.wait() != 0:
LOG.error("Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate())
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field,
df):
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(
_("Instance directory %s already exists! Please remove it from the file system."
) % instancedir)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df,
False)
try:
shutil.move(solr_config_path, instancedir)
finally:
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
# Remove the root node from Zookeeper
zc.delete_path(root_node)
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
finally:
# Remove tmp config directory
shutil.rmtree(tmp_path)
else: # Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
try:
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
shutil.move(solr_config_path, instancedir)
shutil.rmtree(tmp_path)
if not self.api.create_core(name, instancedir):
raise Exception('Failed to create core: %s' % name)
except Exception, e:
raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
finally:
# Delete instance directory if we couldn't create the core.
shutil.rmtree(instancedir)
return name
#
def delete_index(self, name):
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
else:
# Non-solrcloud mode
# Create instance directory locally.
instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name)
if os.path.exists(instancedir):
raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir)
tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False)
shutil.move(solr_config_path, instancedir)
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_core(name, instancedir):
# Delete instance directory if we couldn't create a collection.
shutil.rmtree(instancedir)
raise PopupException(_('Could not create collection. Check error logs for more info.'))
def delete_collection(self, name, core):
"""
Delete solr collection/core and instance dir
"""
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if core:
