def service_check(self, env):
"""
Performs a service check for Indexing.
:param env: Environment
"""
metron_service.check_indexer_parameters()
Logger.info('Checking Kafka topics for Indexing')
metron_service.check_kafka_topics(self.__params, self.__get_topics())
Logger.info("Checking HBase for Indexing")
metron_service.check_hbase_table(self.__params, self.__params.update_hbase_table)
metron_service.check_hbase_column_family(self.__params, self.__params.update_hbase_table, self.__params.update_hbase_cf)
Logger.info('Checking Elasticsearch templates for Indexing')
self.check_elasticsearch_templates()
if self.__params.security_enabled:
Logger.info('Checking Kafka ACLs for Indexing')
metron_service.check_kafka_acls(self.__params, self.__get_topics())
metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
Logger.info("Checking HBase ACLs for Indexing")
metron_service.check_hbase_acls(self.__params, self.__params.update_hbase_table)
Logger.info("Checking for Indexing topology")
if not self.is_topology_active(env):
raise Fail("Indexing topology not running")
Logger.info("Indexing service check completed successfully")
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
Logger.info(
ambari_format(
'Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'
))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
ses = requests.session()
ses = commands.get_zeppelin_auth_details(ses,
params.zeppelin_server_url,
env)
for dirName, subdirList, files in os.walk(
params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
Logger.info("Importing notebook: " + fileName)
zeppelin_import_url = ambari_format(
'http://{zeppelin_server_url}/api/notebook/import')
zeppelin_notebook = {
'file': open(os.path.join(dirName, fileName), 'rb')
}
res = ses.post(zeppelin_import_url,
files=zeppelin_notebook)
Logger.info("Result: " + res.text)
def kibana_dashboard_install(self, env):
from params import params
env.set_params(params)
metron_service.check_indexer_parameters()
Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
if not os.path.isfile(kibanaTemplate):
raise IOError(
errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
template_cmd = ambari_format(
'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate)
Execute(template_cmd, logoutput=True)
kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
if not os.path.isfile(kibanaDashboardLoad):
raise IOError(
errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
kibana_cmd = ambari_format(
'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
Execute(kibana_cmd, logoutput=True)
def elasticsearch_template_delete(self, env):
from params import params
env.set_params(params)
Logger.info("Deleting Elasticsearch index templates")
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name in commands.get_templates():
# delete the index template
cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""
Execute(cmd.format(params.es_http_url, template_name),
logoutput=True)
def elasticsearch_template_delete(self, env):
from params import params
env.set_params(params)
Logger.info("Deleting Elasticsearch index templates")
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name in commands.get_templates():
# delete the index template
cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""
Execute(
cmd.format(params.es_http_url, template_name),
logoutput=True)
def configure(self, env, upgrade_type=None, config_dir=None):
from params import params
env.set_params(params)
Logger.info("Running indexing configure")
metron_service.check_indexer_parameters()
File(format("{metron_config_path}/elasticsearch.properties"),
content=Template("elasticsearch.properties.j2"),
owner=params.metron_user,
group=params.metron_group
)
File(format("{metron_config_path}/solr.properties"),
content=Template("solr.properties.j2"),
owner=params.metron_user,
group=params.metron_group
)
File(format("{metron_config_path}/hdfs.properties"),
content=Template("hdfs.properties.j2"),
owner=params.metron_user,
group=params.metron_group
)
if not metron_service.is_zk_configured(params):
metron_service.init_zk_config(params)
metron_service.set_zk_configured(params)
metron_service.refresh_configs(params)
commands = IndexingCommands(params)
if not commands.is_configured():
commands.init_kafka_topics()
commands.init_hdfs_dir()
commands.set_configured()
if params.security_enabled and not commands.is_hdfs_perm_configured():
# If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group
# If we start off Kerberized, it just does the same thing twice.
commands.init_hdfs_dir()
commands.set_hdfs_perm_configured()
if params.security_enabled and not commands.is_acl_configured():
commands.init_kafka_acls()
commands.set_acl_configured()
if not commands.is_hbase_configured():
commands.create_hbase_tables()
if params.security_enabled and not commands.is_hbase_acl_configured():
commands.set_hbase_acls()
Logger.info("Calling security setup")
storm_security_setup(params)
def elasticsearch_template_install(self, env):
from params import params
env.set_params(params)
Logger.info("Installing Elasticsearch index templates")
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name, template_path in commands.get_templates().iteritems(
):
# install the index template
File(template_path,
mode=0755,
content=StaticFile("{0}.template".format(template_name)))
cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}"
Execute(cmd.format(params.es_http_url, template_name,
template_path),
logoutput=True)
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
ses = requests.session()
ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env)
for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
Logger.info("Importing notebook: " + fileName)
zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import')
zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')}
res = ses.post(zeppelin_import_url, files=zeppelin_notebook)
Logger.info("Result: " + res.text)
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env)
for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
Logger.info("Importing notebook: " + fileName)
zeppelin_notebook = os.path.join(dirName, fileName)
zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\''
zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook)
return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user)
Logger.info("Status of importing notebook: " + import_result)
if return_code != 0:
Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
def elasticsearch_template_install(self, env):
from params import params
env.set_params(params)
Logger.info("Installing Elasticsearch index templates")
try:
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name, template_path in commands.get_templates().iteritems():
# install the index template
File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name)))
cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}"
Execute(
cmd.format(params.es_http_url, template_name, template_path),
logoutput=True)
return True
except Exception as e:
msg = "WARNING: Elasticsearch index templates could not be installed. " \
"Is Elasticsearch running? Will reattempt install on next start. error={0}"
Logger.warning(msg.format(e))
return False
