def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, True) # Create instance directory. solrctl_path = get_solrctl_path() process = subprocess.Popen([solrctl_path, "instancedir", "--create", name, solr_config_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) status = process.wait() # Don't want directories laying around shutil.rmtree(tmp_path) if status != 0: LOG.error("Could not create instance directory.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException(_('Could not create instance directory. ' 'Check if solr_zk_ensemble and solrctl_path are correct in Hue config [indexer].')) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. process = subprocess.Popen([solrctl_path, "instancedir", "--delete", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) if process.wait() != 0: LOG.error("Cloud not delete collection.\nOutput: %s\nError: %s" % process.communicate()) raise PopupException(_('Could not create collection. Check error logs for more info.')) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(conf.CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException( _('Error in copying Solr configurations.'), detail=e) finally: # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException( _('Error in deleting Solr configurations.'), detail=e) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
def _create_non_solr_cloud_index(self, name, fields, unique_key_field, df): # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException( _("Instance directory %s already exists! Please remove it from the file system." ) % instancedir) try: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) try: shutil.move(solr_config_path, instancedir) finally: shutil.rmtree(tmp_path) if not self.api.create_core(name, instancedir): raise Exception('Failed to create core: %s' % name) except Exception as e: raise PopupException( _('Could not create index. Check error logs for more info.'), detail=e) finally: shutil.rmtree(instancedir)
def _create_cloud_config(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=self.get_zookeeper_host(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs( fields=fields, unique_key_field=unique_key_field, df=df, solr_cloud_mode=True, is_solr_six_or_more=self.is_solr_six_or_more(), is_solr_hdfs_mode=self.is_solr_with_hdfs(), is_sentry_protected=self.is_sentry_protected()) try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') if not zc.path_exists(root_node): zc.copy_path(root_node, config_root_path) else: LOG.warn('Config %s already existing.' % name) except Exception as e: if zc.path_exists(root_node): zc.delete_path(root_node) raise PopupException(_('Could not create index: %s') % e) finally: shutil.rmtree(tmp_path)
def _create_solr_cloud_index(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if is_enabled(): with open( os.path.join(config_root_path, 'solrconfig.xml.secure')) as f: zc.set( os.path.join(root_node, 'conf', 'solrconfig.xml'), f.read()) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_( 'Could not create index. Check error logs for more info.'), detail=e) finally:
def create_index(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_( 'Could not create index. Check error logs for more info.'), detail=e) finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # Need to remove path afterwards tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) # Don't want directories laying around shutil.rmtree(tmp_path) if not self.api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): # solrcloud mode # Need to remove path afterwards tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e)
def _create_solr_cloud_collection(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: config_root_path = '%s/%s' % (solr_config_path, 'conf') try: zc.copy_path(root_node, config_root_path) except Exception, e: zc.delete_path(root_node) raise PopupException(_('Error in copying Solr configurations.'), detail=e) finally: # Don't want directories laying around shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection and instance dir. Create schema.xml file so that we can set UniqueKey field. """ # Need to remove path afterwards tmp_path, solr_config_path = utils.copy_configs( fields, unique_key_field, df) # Create instance directory. process = subprocess.Popen( [ conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_HOME': conf.SOLR_HOME.get(), 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) status = process.wait() shutil.rmtree(tmp_path) if status != 0: LOG.error( "Cloud not create instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate()) raise PopupException( _('Could not create instance directory. Check error logs for more info.' )) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory. process = subprocess.Popen( [conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_HOME': conf.SOLR_HOME.get(), 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) if process.wait() != 0: LOG.error( "Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate()) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
def _create_solr_cloud_index(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e) finally:
def _create_non_solr_cloud_index(self, name, fields, unique_key_field, df): # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) try: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) try: shutil.move(solr_config_path, instancedir) finally: shutil.rmtree(tmp_path) if not self.api.create_core(name, instancedir): raise Exception('Failed to create core: %s' % name) except Exception, e: raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e)
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field, df): # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) try: shutil.move(solr_config_path, instancedir) finally: shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_solr_cloud_index(self, name, fields, unique_key_field, df): with ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) as zc: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: root_node = "%s/%s" % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = "%s/%s" % (solr_config_path, "conf") zc.copy_path(root_node, config_root_path) if is_enabled(): with open(os.path.join(config_root_path, "solrconfig.xml.secure")) as f: zc.set(os.path.join(root_node, "conf", "solrconfig.xml"), f.read()) if not self.api.create_collection(name): raise Exception("Failed to create collection: %s" % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_("Could not create index. Check error logs for more info."), detail=e) finally:
def create_index(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection or core and instance dir. Create schema.xml file so that we can set UniqueKey field. """ if self.is_solr_cloud_mode(): tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, True) try: zc = ZookeeperClient(hosts=get_solr_ensemble(), read_only=False) root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name) config_root_path = '%s/%s' % (solr_config_path, 'conf') zc.copy_path(root_node, config_root_path) if not self.api.create_collection(name): raise Exception('Failed to create collection: %s' % name) except Exception, e: if zc.path_exists(root_node): # Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e) finally:
def create_collection(self, name, fields, unique_key_field='id', df='text'): """ Create solr collection and instance dir. Create schema.xml file so that we can set UniqueKey field. """ # Need to remove path afterwards tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df) # Create instance directory. process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_HOME': conf.SOLR_HOME.get(), 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) status = process.wait() shutil.rmtree(tmp_path) if status != 0: LOG.error("Cloud not create instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate()) raise PopupException(_('Could not create instance directory. Check error logs for more info.')) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory. process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--delete", name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ 'SOLR_HOME': conf.SOLR_HOME.get(), 'SOLR_ZK_ENSEMBLE': conf.SOLR_ZK_ENSEMBLE.get() }) if process.wait() != 0: LOG.error("Cloud not delete instance directory.\nOutput stream: %s\nError stream: %s" % process.communicate()) raise PopupException(_('Could not create collection. Check error logs for more info.'))
def _create_non_solr_cloud_collection(self, name, fields, unique_key_field, df): # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException( _("Instance directory %s already exists! Please remove it from the file system." ) % instancedir) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) try: shutil.move(solr_config_path, instancedir) finally: shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException( _('Could not create collection. Check error logs for more info.' ))
# Remove the root node from Zookeeper zc.delete_path(root_node) raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e) finally: # Remove tmp config directory shutil.rmtree(tmp_path) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) try: tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) if not self.api.create_core(name, instancedir): raise Exception('Failed to create core: %s' % name) except Exception, e: raise PopupException(_('Could not create index. Check error logs for more info.'), detail=e) finally: # Delete instance directory if we couldn't create the core. shutil.rmtree(instancedir) return name # def delete_index(self, name):
shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_collection(name): # Delete instance directory if we couldn't create a collection. try: zc.delete_path(root_node) except Exception, e: raise PopupException(_('Error in deleting Solr configurations.'), detail=e) else: # Non-solrcloud mode # Create instance directory locally. instancedir = os.path.join(CORE_INSTANCE_DIR.get(), name) if os.path.exists(instancedir): raise PopupException(_("Instance directory %s already exists! Please remove it from the file system.") % instancedir) tmp_path, solr_config_path = copy_configs(fields, unique_key_field, df, False) shutil.move(solr_config_path, instancedir) shutil.rmtree(tmp_path) api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if not api.create_core(name, instancedir): # Delete instance directory if we couldn't create a collection. shutil.rmtree(instancedir) raise PopupException(_('Could not create collection. Check error logs for more info.')) def delete_collection(self, name, core): """ Delete solr collection/core and instance dir """ api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get()) if core: