def service_check(self, env): """ Performs a service check for Indexing. :param env: Environment """ metron_service.check_indexer_parameters() Logger.info('Checking Kafka topics for Indexing') metron_service.check_kafka_topics(self.__params, self.__get_topics()) Logger.info("Checking HBase for Indexing") metron_service.check_hbase_table(self.__params, self.__params.update_hbase_table) metron_service.check_hbase_column_family(self.__params, self.__params.update_hbase_table, self.__params.update_hbase_cf) Logger.info('Checking Elasticsearch templates for Indexing') self.check_elasticsearch_templates() if self.__params.security_enabled: Logger.info('Checking Kafka ACLs for Indexing') metron_service.check_kafka_acls(self.__params, self.__get_topics()) metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups()) Logger.info("Checking HBase ACLs for Indexing") metron_service.check_hbase_acls(self.__params, self.__params.update_hbase_table) Logger.info("Checking for Indexing topology") if not self.is_topology_active(env): raise Fail("Indexing topology not running") Logger.info("Indexing service check completed successfully")
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) metron_service.check_indexer_parameters() commands = IndexingCommands(params) Logger.info( ambari_format( 'Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}' )) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. ses = requests.session() ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk( params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_import_url = ambari_format( 'http://{zeppelin_server_url}/api/notebook/import') zeppelin_notebook = { 'file': open(os.path.join(dirName, fileName), 'rb') } res = ses.post(zeppelin_import_url, files=zeppelin_notebook) Logger.info("Result: " + res.text)
def kibana_dashboard_install(self, env): from params import params env.set_params(params) metron_service.check_indexer_parameters() Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url)) kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') if not os.path.isfile(kibanaTemplate): raise IOError( errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) Logger.info("Loading .kibana index template from %s" % kibanaTemplate) template_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate) Execute(template_cmd, logoutput=True) kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') if not os.path.isfile(kibanaDashboardLoad): raise IOError( errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) kibana_cmd = ambari_format( 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) Execute(kibana_cmd, logoutput=True)
def elasticsearch_template_delete(self, env): from params import params env.set_params(params) Logger.info("Deleting Elasticsearch index templates") metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name in commands.get_templates(): # delete the index template cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\"" Execute(cmd.format(params.es_http_url, template_name), logoutput=True)
def elasticsearch_template_delete(self, env): from params import params env.set_params(params) Logger.info("Deleting Elasticsearch index templates") metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name in commands.get_templates(): # delete the index template cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\"" Execute( cmd.format(params.es_http_url, template_name), logoutput=True)
def configure(self, env, upgrade_type=None, config_dir=None): from params import params env.set_params(params) Logger.info("Running indexing configure") metron_service.check_indexer_parameters() File(format("{metron_config_path}/elasticsearch.properties"), content=Template("elasticsearch.properties.j2"), owner=params.metron_user, group=params.metron_group ) File(format("{metron_config_path}/solr.properties"), content=Template("solr.properties.j2"), owner=params.metron_user, group=params.metron_group ) File(format("{metron_config_path}/hdfs.properties"), content=Template("hdfs.properties.j2"), owner=params.metron_user, group=params.metron_group ) if not metron_service.is_zk_configured(params): metron_service.init_zk_config(params) metron_service.set_zk_configured(params) metron_service.refresh_configs(params) commands = IndexingCommands(params) if not commands.is_configured(): commands.init_kafka_topics() commands.init_hdfs_dir() commands.set_configured() if params.security_enabled and not commands.is_hdfs_perm_configured(): # If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group # If we start off Kerberized, it just does the same thing twice. commands.init_hdfs_dir() commands.set_hdfs_perm_configured() if params.security_enabled and not commands.is_acl_configured(): commands.init_kafka_acls() commands.set_acl_configured() if not commands.is_hbase_configured(): commands.create_hbase_tables() if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() Logger.info("Calling security setup") storm_security_setup(params)
def elasticsearch_template_install(self, env): from params import params env.set_params(params) Logger.info("Installing Elasticsearch index templates") metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name, template_path in commands.get_templates().iteritems( ): # install the index template File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name))) cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}" Execute(cmd.format(params.es_http_url, template_name, template_path), logoutput=True)
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) metron_service.check_indexer_parameters() commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. ses = requests.session() ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import') zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')} res = ses.post(zeppelin_import_url, files=zeppelin_notebook) Logger.info("Result: " + res.text)
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) metron_service.check_indexer_parameters() commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_notebook = os.path.join(dirName, fileName) zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\'' zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook) return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user) Logger.info("Status of importing notebook: " + import_result) if return_code != 0: Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
def elasticsearch_template_install(self, env): from params import params env.set_params(params) Logger.info("Installing Elasticsearch index templates") try: metron_service.check_indexer_parameters() commands = IndexingCommands(params) for template_name, template_path in commands.get_templates().iteritems(): # install the index template File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name))) cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}" Execute( cmd.format(params.es_http_url, template_name, template_path), logoutput=True) return True except Exception as e: msg = "WARNING: Elasticsearch index templates could not be installed. " \ "Is Elasticsearch running? Will reattempt install on next start. error={0}" Logger.warning(msg.format(e)) return False