def test_get_ensemble_upstream_solr(self):
client = SolrClient(self.user, api=MockSolrUpstreamCloudApi())
client._reset_properties()
assert_true(client.is_solr_cloud_mode())
assert_true(client.is_solr_six_or_more())
assert_false(client.is_solr_with_hdfs())
assert_equal('localhost:9983', client.get_zookeeper_host())
def test_get_ensemble_cdh_solr(self):
client = SolrClient(self.user, api=MockSolrCdhCloudHdfsApi())
client._reset_properties()
assert_true(client.is_solr_cloud_mode())
assert_false(client.is_solr_six_or_more())
assert_true(client.is_solr_with_hdfs())
assert_equal('hue.com:2181/solr', client.get_zookeeper_host())
def run_morphline(self, request, collection_name, morphline, input_path, query=None, start_time=None, lib_path=None):
workspace_path = self._upload_workspace(morphline)
task = make_notebook(
name=_('Indexing into %s') % collection_name,
editor_type='notebook',
on_success_url=reverse('search:browse', kwargs={'name': collection_name}),
pub_sub_url='assist.collections.refresh',
is_task=True,
is_notebook=True,
last_executed=start_time
)
if query:
q = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=query))
notebook_data = q.get_data()
snippet = notebook_data['snippets'][0]
api = get_api(request, snippet)
destination = '__hue_%s' % notebook_data['uuid'][:4]
location = '/user/%s/__hue-%s' % (request.user, notebook_data['uuid'][:4])
sql, _success_url = api.export_data_as_table(notebook_data, snippet, destination, is_temporary=True, location=location)
input_path = '${nameNode}%s' % location
task.add_hive_snippet(snippet['database'], sql)
client = SolrClient(self.user)
extra_args = ['-Dmapreduce.job.user.classpath.first=true'] if client.is_solr_six_or_more() else []
task.add_java_snippet(
clazz='org.apache.solr.hadoop.MapReduceIndexerTool',
app_jar=lib_path if lib_path is not None else CONFIG_INDEXER_LIBS_PATH.get(),
arguments=extra_args + [
u'--morphline-file',
u'morphline.conf',
u'--output-dir',
u'${nameNode}/user/%s/indexer' % self.username,
u'--log4j',
u'log4j.properties',
u'--go-live',
u'--zk-host',
client.get_zookeeper_host(),
u'--collection',
collection_name,
input_path,
],
files=[
{u'path': u'%s/log4j.properties' % workspace_path, u'type': u'file'},
{u'path': u'%s/morphline.conf' % workspace_path, u'type': u'file'}
]
)
return task.execute(request, batch=True)
def _create_solr_cloud_collection(self, name, fields, unique_key_field,
df):
client = SolrClient(self.user)
with ZookeeperClient(hosts=client.get_zookeeper_host(),
read_only=False) as zc:
root_node = '%s/%s' % (ZK_SOLR_CONFIG_NAMESPACE, name)
tmp_path, solr_config_path = copy_configs(
fields=fields,
unique_key_field=unique_key_field,
df=df,
solr_cloud_mode=client.is_solr_cloud_mode(),
is_solr_six_or_more=client.is_solr_six_or_more(),
is_solr_hdfs_mode=client.is_solr_with_hdfs())
try:
config_root_path = '%s/%s' % (solr_config_path, 'conf')
try:
zc.copy_path(root_node, config_root_path)
except Exception, e:
zc.delete_path(root_node)
raise PopupException(
_('Error in copying Solr configurations: %s') % e)
finally:
# Don't want directories laying around
shutil.rmtree(tmp_path)
api = SolrApi(SOLR_URL.get(), self.user, SECURITY_ENABLED.get())
if not api.create_collection(name):
# Delete instance directory if we couldn't create a collection.
try:
zc.delete_path(root_node)
except Exception, e:
raise PopupException(
_('Error in deleting Solr configurations.'), detail=e)
raise PopupException(
_('Could not create collection. Check error logs for more info.'
))
