def pull_config(params): Logger.info('Pulling all Metron configs down from ZooKeeper to local file system') Logger.info('NOTE - THIS IS OVERWRITING THE LOCAL METRON CONFIG DIR WITH ZOOKEEPER CONTENTS: ' + params.metron_zookeeper_config_path) Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PULL --output_dir {metron_zookeeper_config_path} --force"), path=ambari_format("{java_home}/bin") )
def elasticsearch_template_install(self, env): from params import params env.set_params(params) File(params.bro_index_path, mode=0755, content=StaticFile('bro_index.template') ) File(params.snort_index_path, mode=0755, content=StaticFile('snort_index.template') ) File(params.yaf_index_path, mode=0755, content=StaticFile('yaf_index.template') ) bro_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}') Execute(bro_cmd, logoutput=True) snort_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}') Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}') Execute(yaf_cmd, logoutput=True)
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) commands = IndexingCommands(params) Logger.info( ambari_format( 'Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}' )) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. ses = requests.session() ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk( params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_import_url = ambari_format( 'http://{zeppelin_server_url}/api/notebook/import') zeppelin_notebook = { 'file': open(os.path.join(dirName, fileName), 'rb') } res = ses.post(zeppelin_import_url, files=zeppelin_notebook) Logger.info("Result: " + res.text)
def kibana_dashboard_install(self, env): from params import params env.set_params(params) Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url)) kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') if not os.path.isfile(kibanaTemplate): raise IOError( errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) Logger.info("Loading .kibana index template from %s" % kibanaTemplate) template_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate) Execute(template_cmd, logoutput=True) kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') if not os.path.isfile(kibanaDashboardLoad): raise IOError( errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) kibana_cmd = ambari_format( 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) Execute(kibana_cmd, logoutput=True)
def elasticsearch_template_install(self, env): from params import params env.set_params(params) File(params.bro_index_path, mode=0755, content=StaticFile('bro_index.template')) File(params.snort_index_path, mode=0755, content=StaticFile('snort_index.template')) File(params.yaf_index_path, mode=0755, content=StaticFile('yaf_index.template')) bro_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}' ) Execute(bro_cmd, logoutput=True) snort_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}' ) Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format( 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}' ) Execute(yaf_cmd, logoutput=True)
def init_zk_config(params): Logger.info( 'Loading ALL Metron config into ZooKeeper - this command should ONLY be executed by Ambari on initial install.' ) Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PUSH --input_dir {metron_zookeeper_config_path}" ), path=ambari_format("{java_home}/bin"))
def metron_knox_topology_setup(params): if os.path.exists(params.knox_home): File(ambari_format("{knox_home}/conf/topologies/metron.xml"), content=Template("metron.xml.j2"), owner=params.knox_user, group=params.knox_group) File(ambari_format("{knox_home}/conf/topologies/metronsso.xml"), content=Template("metronsso.xml.j2"), owner=params.knox_user, group=params.knox_group)
def patch_global_config(params): patch_file = "/tmp/metron-global-config-patch.json" Logger.info("Setup temporary global config JSON patch (formatting per RFC6902): " + patch_file) build_global_config_patch(params, patch_file) Logger.info('Patching global config in ZooKeeper') Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file), path=ambari_format("{java_home}/bin") )
def elasticsearch_template_delete(self, env): from params import params env.set_params(params) bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"') Execute(bro_cmd, logoutput=True) snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"') Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"') Execute(yaf_cmd, logoutput=True)
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): zeppelin_cmd = ambari_format( 'curl -s -XPOST http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, fileName) + '"') Execute(zeppelin_cmd, logoutput=True)
def metron_knox_topology_setup(params): if os.path.exists(params.knox_home): File(ambari_format("{knox_home}/conf/topologies/metron.xml"), content=Template("metron.xml.j2"), owner=params.knox_user, group=params.knox_group ) File(ambari_format("{knox_home}/conf/topologies/metronsso.xml"), content=Template("metronsso.xml.j2"), owner=params.knox_user, group=params.knox_group )
def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env): """ With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the session connection details to be used for importing Zeppelin notebooks. :param ses: Session handle :param zeppelin_server_url: Zeppelin Server URL :return: ses """ from params import params env.set_params(params) # Check if authentication is enabled on the Zeppelin server try: ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) # Establish connection if authentication is enabled try: Logger.info( "Shiro authentication is found to be enabled on the Zeppelin server." ) # Read the Shiro admin user credentials from Zeppelin config in Ambari seen_users = False username = None password = None if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): seen_users = True tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() userpassword = tokens.split(',')[0].strip() username = userpassword.split('=')[0].strip() password = userpassword.split('=')[1].strip() else: Logger.error( "ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail." ) zeppelin_payload = {'userName': username, 'password': password} ses.post( ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) except: pass # If authentication is not enabled, fall back to default method of imporing notebooks except requests.exceptions.RequestException: ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) return ses
def elasticsearch_template_delete(self, env): from params import params env.set_params(params) bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"') Execute(bro_cmd, logoutput=True) snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"') Execute(snort_cmd, logoutput=True) yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"') Execute(yaf_cmd, logoutput=True) error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/error_index*"') Execute(error_cmd, logoutput=True)
def storm_security_setup(params): if params.security_enabled: # I don't think there's an Ambari way to get a user's local home dir , so have Python perform tilde expansion. # Ambari's Directory doesn't do tilde expansion. metron_storm_dir_tilde = '~' + params.metron_user + '/.storm' metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde) Directory(params.metron_home, mode=0755, owner=params.metron_user, group=params.metron_group, create_parents=True) Directory(metron_storm_dir, mode=0755, owner=params.metron_user, group=params.metron_group) File(ambari_format('{client_jaas_path}'), content=InlineTemplate(params.metron_client_jaas_conf_template), owner=params.metron_user, group=params.metron_group, mode=0755) File(metron_storm_dir + '/storm.yaml', content=Template('storm.yaml.j2'), owner=params.metron_user, group=params.metron_group, mode=0755) File(metron_storm_dir + '/storm.config', content=Template('storm.config.j2'), owner=params.metron_user, group=params.metron_group, mode=0755)
def get_running_topologies(params): Logger.info('Getting Running Storm Topologies from Storm REST Server') Logger.info('Security enabled? ' + str(params.security_enabled)) # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets. # This is becuase we need to run a command with a return as the metron user. Sigh negotiate = '--negotiate -u : ' if params.security_enabled else '' cmd = ambari_format('curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) Logger.info('Running cmd: ' + cmd) return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user, is_checked_call=False) if (return_code != 0): return {} try: stormjson = json.loads(stdout) except ValueError, e: Logger.info('Stdout: ' + str(stdout)) Logger.info('Stderr: ' + str(stderr)) Logger.exception(str(e)) return {}
def get_running_topologies(params): Logger.info('Getting Running Storm Topologies from Storm REST Server') Logger.info('Security enabled? ' + str(params.security_enabled)) # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets. # This is becuase we need to run a command with a return as the metron user. Sigh negotiate = '--negotiate -u : ' if params.security_enabled else '' cmd = ambari_format( 'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) Logger.info('Running cmd: ' + cmd) return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user, is_checked_call=False) if (return_code != 0): return {} try: stormjson = json.loads(stdout) except ValueError, e: Logger.info('Stdout: ' + str(stdout)) Logger.info('Stderr: ' + str(stderr)) Logger.exception(str(e)) return {}
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. ses = requests.session() ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import') zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')} res = ses.post(zeppelin_import_url, files=zeppelin_notebook) Logger.info("Result: " + res.text)
def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env): """ With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the session connection details to be used for importing Zeppelin notebooks. :param ses: Session handle :param zeppelin_server_url: Zeppelin Server URL :return: ses """ from params import params env.set_params(params) # Check if authentication is enabled on the Zeppelin server try: ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) # Establish connection if authentication is enabled try: Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") # Read the Shiro admin user credentials from Zeppelin config in Ambari seen_users = False username = None password = None if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): seen_users = True tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() userpassword = tokens.split(',')[0].strip() username = userpassword.split('=')[0].strip() password = userpassword.split('=')[1].strip() else: Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") zeppelin_payload = {'userName': username, 'password' : password} ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) except: pass # If authentication is not enabled, fall back to default method of imporing notebooks except requests.exceptions.RequestException: ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) return ses
def load_global_config(params): Logger.info('Create Metron Local Config Directory') Logger.info("Configure Metron global.json") directories = [params.metron_zookeeper_config_path] Directory(directories, mode=0755, owner=params.metron_user, group=params.metron_group) File(ambari_format("{metron_zookeeper_config_path}/global.json"), content=Template("global.json.j2"), owner=params.metron_user, group=params.metron_group) init_config()
def get_running_topologies(): Logger.info('Getting Running Storm Topologies from Storm REST Server') cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary') proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = proc.communicate() try: stormjson = json.loads(stdout) except ValueError: return {} topologiesDict = {} for topology in stormjson['topologies']: topologiesDict[topology['name']] = topology['status'] Logger.info("Topologies: " + str(topologiesDict)) return topologiesDict
def storm_security_setup(params): if params.security_enabled: # I don't think there's an Ambari way to get a user's local home dir , so have Python perform tilde expansion. # Ambari's Directory doesn't do tilde expansion. metron_storm_dir_tilde = '~' + params.metron_user + '/.storm' metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde) Directory(params.metron_home, mode=0755, owner=params.metron_user, group=params.metron_group, create_parents=True ) Directory(metron_storm_dir, mode=0755, owner=params.metron_user, group=params.metron_group ) File(ambari_format('{client_jaas_path}'), content=Template('client_jaas.conf.j2'), owner=params.metron_user, group=params.metron_group, mode=0755 ) File(metron_storm_dir + '/storm.yaml', content=Template('storm.yaml.j2'), owner=params.metron_user, group=params.metron_group, mode=0755 ) File(metron_storm_dir + '/storm.config', content=Template('storm.config.j2'), owner=params.metron_user, group=params.metron_group, mode=0755 )
def zeppelin_notebook_import(self, env): from params import params env.set_params(params) metron_service.check_indexer_parameters() commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) zeppelin_notebook = os.path.join(dirName, fileName) zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\'' zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook) return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user) Logger.info("Status of importing notebook: " + import_result) if return_code != 0: Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
def init_config(): Logger.info('Loading config into ZooKeeper') Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"), path=ambari_format("{java_home}/bin") )
def build_global_config_patch(params, patch_file): """ Build the file used to patch the global configuration. See RFC 6902 at https://tools.ietf.org/html/rfc6902 :param params: :param patch_file: The path where the patch file will be created. """ if params.ra_indexing_writer == 'Solr': indexing_patches = solr_global_config_patches() else: indexing_patches = elasticsearch_global_config_patches() other_patches = """ { "op": "add", "path": "/profiler.client.period.duration", "value": "{{profiler_period_duration}}" }, { "op": "add", "path": "/profiler.client.period.duration.units", "value": "{{profiler_period_units}}" }, { "op": "add", "path": "/parser.error.topic", "value": "{{parser_error_topic}}" }, { "op": "add", "path": "/update.hbase.table", "value": "{{update_hbase_table}}" }, { "op": "add", "path": "/update.hbase.cf", "value": "{{update_hbase_cf}}" }, { "op": "add", "path": "/user.settings.hbase.table", "value": "{{user_settings_hbase_table}}" }, { "op": "add", "path": "/user.settings.hbase.cf", "value": "{{user_settings_hbase_cf}}" }, { "op": "add", "path": "/bootstrap.servers", "value": "{{kafka_brokers}}" }, { "op": "add", "path": "/source.type.field", "value": "{{source_type_field}}" }, { "op": "add", "path": "/threat.triage.score.field", "value": "{{threat_triage_score_field}}" }, { "op": "add", "path": "/enrichment.writer.batchSize", "value": "{{enrichment_kafka_writer_batch_size}}" }, { "op": "add", "path": "/enrichment.writer.batchTimeout", "value": "{{enrichment_kafka_writer_batch_timeout}}" }, { "op": "add", "path": "/profiler.writer.batchSize", "value": "{{profiler_kafka_writer_batch_size}}" }, { "op": "add", "path": "/profiler.writer.batchTimeout", "value": "{{profiler_kafka_writer_batch_timeout}}" } """ patch_template = ambari_format(""" [ {indexing_patches}, {other_patches} ] """) File(patch_file, content=InlineTemplate(patch_template), owner=params.metron_user, group=params.metron_group)
def init_config(): Logger.info('Loading config into ZooKeeper') Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}" ), path=ambari_format("{java_home}/bin"))
def build_global_config_patch(params, patch_file): """ Build the file used to patch the global configuration. See RFC 6902 at https://tools.ietf.org/html/rfc6902 :param params: :param patch_file: The path where the patch file will be created. """ if params.ra_indexing_writer == 'Solr': indexing_patches = solr_global_config_patches() else: indexing_patches = elasticsearch_global_config_patches() other_patches = """ { "op": "add", "path": "/profiler.client.period.duration", "value": "{{profiler_period_duration}}" }, { "op": "add", "path": "/profiler.client.period.duration.units", "value": "{{profiler_period_units}}" }, { "op": "add", "path": "/parser.error.topic", "value": "{{parser_error_topic}}" }, { "op": "add", "path": "/update.hbase.table", "value": "{{update_hbase_table}}" }, { "op": "add", "path": "/update.hbase.cf", "value": "{{update_hbase_cf}}" }, { "op": "add", "path": "/user.settings.hbase.table", "value": "{{user_settings_hbase_table}}" }, { "op": "add", "path": "/user.settings.hbase.cf", "value": "{{user_settings_hbase_cf}}" }, { "op": "add", "path": "/bootstrap.servers", "value": "{{kafka_brokers}}" }, { "op": "add", "path": "/source.type.field", "value": "{{source_type_field}}" }, { "op": "add", "path": "/threat.triage.score.field", "value": "{{threat_triage_score_field}}" }, { "op": "add", "path": "/enrichment.writer.batchSize", "value": "{{enrichment_kafka_writer_batch_size}}" }, { "op": "add", "path": "/enrichment.writer.batchTimeout", "value": "{{enrichment_kafka_writer_batch_timeout}}" }, { "op": "add", "path": "/profiler.writer.batchSize", "value": "{{profiler_kafka_writer_batch_size}}" }, { "op": "add", "path": "/profiler.writer.batchTimeout", "value": "{{profiler_kafka_writer_batch_timeout}}" } """ patch_template = ambari_format( """ [ {indexing_patches}, {other_patches} ] """) File(patch_file, content=InlineTemplate(patch_template), owner=params.metron_user, group=params.metron_group)
def init_zk_config(params): Logger.info('Loading ALL Metron config into ZooKeeper - this command should ONLY be executed by Ambari on initial install.') Execute(ambari_format( "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PUSH --input_dir {metron_zookeeper_config_path}"), path=ambari_format("{java_home}/bin") )