def check_kafka_topics(params, topics): """ Validates that the Kafka topics exist. An exception is raised if any of the topics do not exist. :param params: :param topics: A list of topic names. """ # if needed kinit as 'metron' if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) template = """{0}/kafka-topics.sh \ --zookeeper {1} \ --list | \ awk 'BEGIN {{cnt=0;}} /{2}/ {{cnt++}} END {{if (cnt > 0) {{exit 0}} else {{exit 1}}}}'""" for topic in topics: Logger.info("Checking existence of Kafka topic '{0}'".format(topic)) cmd = template.format(params.kafka_bin_dir, params.zookeeper_quorum, topic) err_msg = "Missing Kafka topic; topic={0}".format(topic) execute(cmd, user=params.kafka_user, err_msg=err_msg)
def solr_schema_install(self, env): from params import params env.set_params(params) Logger.info("Installing Solr schemas") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.solr_keytab_path, self.__params.solr_principal_name, self.__params.solr_user) try: commands = IndexingCommands(params) for collection_name in commands.get_solr_schemas(): # install the schema cmd = format(( "export ZOOKEEPER={solr_zookeeper_url};" "export SECURITY_ENABLED={security_enabled};" )) cmd += "{0}/bin/create_collection.sh {1};" Execute( cmd.format(params.metron_home, collection_name), user=self.__params.solr_user) return True except Exception as e: msg = "WARNING: Solr schemas could not be installed. " \ "Is Solr running? Will reattempt install on next start. error={0}" Logger.warning(msg.format(e)) return False
def solr_schema_install(self, env): from params import params env.set_params(params) Logger.info("Installing Solr schemas") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) try: commands = IndexingCommands(params) for collection_name in commands.get_solr_schemas(): # install the schema cmd = format(("export ZOOKEEPER={solr_zookeeper_url};" "export SECURITY_ENABLED={security_enabled};")) cmd += "{0}/bin/create_collection.sh {1};" Execute(cmd.format(params.metron_home, collection_name), user=self.__params.metron_user) return True except Exception as e: msg = "WARNING: Solr schemas could not be installed. " \ "Is Solr running? Will reattempt install on next start. error={0}" Logger.warning(msg.format(e)) return False
def check_hbase_acls(params, table, user=None, permissions="READ,WRITE"): """ Validates that HBase table permissions exist for a user. An exception is raised if the permissions do not exist. :param params: :param table: The name of the HBase table. :param user: The name of the user. :param permissions: The permissions that should exist. """ if user is None: user = params.metron_user Logger.info("Checking HBase ACLs; table={0}, user={1}, permissions={2}".format(table, user, permissions)) # if needed kinit as 'hbase' if params.security_enabled: kinit(params.kinit_path_local, params.hbase_keytab_path, params.hbase_principal_name, execute_user=params.hbase_user) template = """echo "user_permission '{0}'" | \ hbase shell -n | \ grep " {1} " | \ grep "actions={2}" """ cmd = template.format(table, user, permissions) err_msg = "Missing HBase access; table={0}, user={1}, permissions={2}".format(table, user, permissions) execute(cmd, user=params.hbase_user, err_msg=err_msg)
def set_hbase_acls(self): Logger.info("Setting HBase ACLs") if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.hbase_keytab_path, self.__params.hbase_principal_name, execute_user=self.__params.hbase_user) cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n" add_enrichment_acl_cmd = cmd.format(self.__params.metron_user, self.__params.enrichment_hbase_table) Execute(add_enrichment_acl_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user ) add_threatintel_acl_cmd = cmd.format(self.__params.metron_user, self.__params.threatintel_hbase_table) Execute(add_threatintel_acl_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user ) Logger.info("Done setting HBase ACLs") self.set_hbase_acl_configured()
def get_running_topologies(params): Logger.info('Getting Running Storm Topologies from Storm REST Server') Logger.info('Security enabled? ' + str(params.security_enabled)) # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets. # This is becuase we need to run a command with a return as the metron user. Sigh negotiate = '--negotiate -u : ' if params.security_enabled else '' cmd = ambari_format('curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) Logger.info('Running cmd: ' + cmd) return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user, is_checked_call=False) if (return_code != 0): return {} try: stormjson = json.loads(stdout) except ValueError, e: Logger.info('Stdout: ' + str(stdout)) Logger.info('Stderr: ' + str(stderr)) Logger.exception(str(e)) return {}
def get_running_topologies(params): Logger.info('Getting Running Storm Topologies from Storm REST Server') Logger.info('Security enabled? ' + str(params.security_enabled)) # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets. # This is becuase we need to run a command with a return as the metron user. Sigh negotiate = '--negotiate -u : ' if params.security_enabled else '' cmd = ambari_format( 'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary') if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) Logger.info('Running cmd: ' + cmd) return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user, is_checked_call=False) if (return_code != 0): return {} try: stormjson = json.loads(stdout) except ValueError, e: Logger.info('Stdout: ' + str(stdout)) Logger.info('Stderr: ' + str(stderr)) Logger.exception(str(e)) return {}
def start_rest_application(self): """ Start the REST application """ Logger.info('Starting REST application') if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) # Get the PID associated with the service pid_file = format("{metron_rest_pid_dir}/{metron_rest_pid}") pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=self.__params.metron_user, is_checked_call=False)[1] process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") # Set the password with env variable instead of param to avoid it showing in ps cmd = format(( "export METRON_JDBC_PASSWORD={metron_jdbc_password!p};" "export JAVA_HOME={java_home};" "export METRON_REST_CLASSPATH={metron_rest_classpath};" "export METRON_INDEX_CP={metron_indexing_classpath};" "export METRON_LOG_DIR={metron_log_dir};" "export METRON_PID_FILE={pid_file};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" )) Execute(cmd, user = self.__params.metron_user, logoutput=True, not_if = process_id_exists_command, timeout=60) Logger.info('Done starting REST application')
def check_hbase_acls(params, table, user=None, permissions="READ,WRITE"): """ Validates that HBase table permissions exist for a user. An exception is raised if the permissions do not exist. :param params: :param table: The name of the HBase table. :param user: The name of the user. :param permissions: The permissions that should exist. """ if user is None: user = params.metron_user Logger.info( "Checking HBase ACLs; table={0}, user={1}, permissions={2}".format( table, user, permissions)) # if needed kinit as 'hbase' if params.security_enabled: kinit(params.kinit_path_local, params.hbase_keytab_path, params.hbase_principal_name, execute_user=params.hbase_user) template = """echo "user_permission '{0}'" | \ hbase shell -n | \ grep " {1} " | \ grep "actions={2}" """ cmd = template.format(table, user, permissions) err_msg = "Missing HBase access; table={0}, user={1}, permissions={2}".format( table, user, permissions) execute(cmd, user=params.hbase_user, err_msg=err_msg)
def start_parser_topologies(self, env): Logger.info("Starting Metron parser topologies: {0}".format( self.__get_aggr_parsers(self.__params))) start_cmd_template = """{0}/bin/start_parser_topology.sh \ -k {1} \ -z {2} \ -s {3} \ -ksp {4}""" if self.__params.security_enabled: # Append the extra configs needed for secured cluster. start_cmd_template = start_cmd_template + ' -e ~' + self.__params.metron_user + '/.storm/storm.config' metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) stopped_parsers = set(self.__get_aggr_parsers( self.__params)) - self.get_running_topology_names(env) Logger.info('Parsers that need started: ' + str(stopped_parsers)) for parser in stopped_parsers: Logger.info('Starting ' + parser) start_cmd = start_cmd_template.format( self.__params.metron_home, self.__params.kafka_brokers, self.__params.zookeeper_quorum, parser, self.__params.kafka_security_protocol) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) Logger.info('Finished starting parser topologies')
def start_parser_topologies(self, env): Logger.info("Starting Metron parser topologies: {0}".format(self.__get_aggr_parsers(self.__params))) start_cmd_template = """{0}/bin/start_parser_topology.sh \ -k {1} \ -z {2} \ -s {3} \ -ksp {4}""" if self.__params.security_enabled: # Append the extra configs needed for secured cluster. start_cmd_template = start_cmd_template + ' -e ~' + self.__params.metron_user + '/.storm/storm.config' metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) stopped_parsers = set(self.__get_aggr_parsers(self.__params)) - self.get_running_topology_names(env) Logger.info('Parsers that need started: ' + str(stopped_parsers)) for parser in stopped_parsers: Logger.info('Starting ' + parser) start_cmd = start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers, self.__params.zookeeper_quorum, parser, self.__params.kafka_security_protocol) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) Logger.info('Finished starting parser topologies')
def start_rest_application(self): Logger.info('Starting REST application') if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) # Get the PID associated with the service pid_file = format("{metron_rest_pid_dir}/{metron_rest_pid}") pid = get_user_call_output.get_user_call_output( format("cat {pid_file}"), user=self.__params.metron_user, is_checked_call=False)[1] process_id_exists_command = format( "ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") # Set the password with env variable instead of param to avoid it showing in ps cmd = format(("export METRON_JDBC_PASSWORD={metron_jdbc_password!p};" "export JAVA_HOME={java_home};" "export METRON_REST_CLASSPATH={metron_rest_classpath};" "export METRON_INDEX_CP={metron_indexing_classpath};" "export METRON_LOG_DIR={metron_log_dir};" "export METRON_PID_FILE={pid_file};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;")) Execute(cmd, user=self.__params.metron_user, logoutput=True, not_if=process_id_exists_command, timeout=60) Logger.info('Done starting REST application')
def start(self, env, upgrade_type=None): from params import params env.set_params(params) self.configure(env) commands = EnrichmentCommands(params) if params.security_enabled: metron_security.kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) metron_service.load_global_config(params) if not commands.is_kafka_configured(): commands.init_kafka_topics() if params.security_enabled and not commands.is_kafka_acl_configured(): commands.init_kafka_acls() if not commands.is_hbase_configured(): commands.create_hbase_tables() if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() if not commands.is_geo_configured(): commands.init_geo() commands.start_enrichment_topology()
def set_hbase_acls(self): Logger.info("Setting HBase ACLs") if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.hbase_keytab_path, self.__params.hbase_principal_name, execute_user=self.__params.hbase_user) cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n" add_enrichment_acl_cmd = cmd.format( self.__params.metron_user, self.__params.enrichment_hbase_table) Execute(add_enrichment_acl_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user) add_threatintel_acl_cmd = cmd.format( self.__params.metron_user, self.__params.threatintel_hbase_table) Execute(add_threatintel_acl_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user) Logger.info("Done setting HBase ACLs") self.set_hbase_acl_configured()
def start_indexing_topology(self, env): Logger.info('Starting ' + self.__indexing_topology) if not self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \ -s {1} \ -z {2}""" start_cmd = start_cmd_template.format( self.__params.metron_home, self.__indexing_topology, self.__params.zookeeper_quorum) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info('Indexing topology already running') Logger.info('Finished starting indexing topology')
def stop_indexing_topology(self): Logger.info('Stopping ' + self.__indexing_topology) stop_cmd = 'storm kill ' + self.__indexing_topology if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) Execute(stop_cmd, user=self.__params.metron_user) Logger.info('Done stopping indexing topologies')
def start_rest_application(self): Logger.info('Starting REST application') # Build the spring options # the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml metron_spring_options = format(" --server.port={metron_rest_port}") if not "vagrant" in self.__params.metron_spring_profiles_active: metron_spring_options += format( " --spring.config.location={metron_home}/config/rest_application.yml" ) if self.__params.metron_spring_profiles_active: metron_spring_options += format( " --spring.profiles.active={metron_spring_profiles_active}") metron_rest_classpath = format( "{hadoop_conf_dir}:{hbase_conf_dir}:{metron_home}/lib/metron-rest-{metron_version}.jar" ) if self.__params.metron_jdbc_client_path: metron_rest_classpath += format(":{metron_jdbc_client_path}") if self.__params.metron_indexing_classpath: metron_rest_classpath += format(":{metron_indexing_classpath}") else: metron_rest_classpath += format( ":{metron_home}/lib/metron-elasticsearch-{metron_version}-uber.jar" ) if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) # Get the PID associated with the service pid_file = format("{metron_rest_pid_dir}/{metron_rest_pid}") pid = get_user_call_output.get_user_call_output( format("cat {pid_file}"), user=self.__params.metron_user, is_checked_call=False)[1] process_id_exists_command = format( "ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") # Set the password with env variable instead of param to avoid it showing in ps cmd = format(( "set -o allexport; source {metron_sysconfig}; set +o allexport;" "export METRON_JDBC_PASSWORD={metron_jdbc_password!p};" "{java_home}/bin/java -cp {metron_rest_classpath} org.apache.metron.rest.MetronRestApplication {metron_spring_options} >> {metron_log_dir}/metron-rest.log 2>&1 & echo $! > {pid_file};" "unset METRON_JDBC_PASSWORD;")) daemon_cmd = cmd Execute(daemon_cmd, user=self.__params.metron_user, logoutput=True, not_if=process_id_exists_command) Logger.info('Done starting REST application')
def stop_parser_topologies(self): Logger.info('Stopping parsers') for parser in self.get_parser_list(): Logger.info('Stopping ' + parser) stop_cmd = 'storm kill ' + parser if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) Execute(stop_cmd, user=self.__params.metron_user) Logger.info('Done stopping parser topologies')
def status(self, env): from params import status_params env.set_params(status_params) commands = EnrichmentCommands(status_params) if status_params.security_enabled: metron_security.kinit(status_params.kinit_path_local, status_params.metron_keytab_path, status_params.metron_principal_name, execute_user=status_params.metron_user) if not commands.is_topology_active(env): raise ComponentIsNotRunning()
def stop(self, env, upgrade_type=None): from params import params env.set_params(params) commands = EnrichmentCommands(params) if params.security_enabled: metron_security.kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) commands.stop_enrichment_topology()
def stop(self, env, upgrade_type=None): from params import params env.set_params(params) commands = EnrichmentCommands(params) if params.security_enabled: metron_security.kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) commands.stop_enrichment_topology(env)
def stop_rest_application(self): """ Stop the REST application """ Logger.info('Stopping REST application') # Get the pid associated with the service pid_file = format("{metron_rest_pid_dir}/{metron_rest_pid}") pid = get_user_call_output.get_user_call_output( format("cat {pid_file}"), user=self.__params.metron_user, is_checked_call=False)[1] process_id_exists_command = format( "ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) # Politely kill kill_cmd = ('kill', format("{pid}")) Execute(kill_cmd, sudo=True, not_if=format("! ({process_id_exists_command})")) # Violently kill hard_kill_cmd = ('kill', '-9', format("{pid}")) wait_time = 5 Execute( hard_kill_cmd, not_if=format( "! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )" ), sudo=True, ignore_failures=True) try: # check if stopped the process, else fail the task Execute( format("! ({process_id_exists_command})"), tries=20, try_sleep=3, ) except: show_logs(self.__params.metron_log_dir, self.__params.metron_user) raise File(pid_file, action="delete") Logger.info('Done stopping REST application')
def start_indexing_topology(self): Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing)) start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \ -s {1} \ -z {2}""" Logger.info('Starting ' + self.__indexing) if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum), user=self.__params.metron_user) Logger.info('Finished starting indexing topology')
def start_pcap_topology(self, env): Logger.info('Starting Metron PCAP topology') start_cmd_template = """{0}/bin/start_pcap_topology.sh""" if not self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd = start_cmd_template.format(self.__params.metron_home) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else : Logger.info('PCAP topology already started') Logger.info('Finished starting pcap topologies')
def stop_pcap_topology(self, env): Logger.info('Stopping Metron PCAP topology') if self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) stop_cmd = 'storm kill ' + self.__pcap_topology Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else : Logger.info('PCAP topology already stopped') Logger.info('Finished starting PCAP topologies')
def stop_parser_topologies(self, env): Logger.info('Stopping parsers') running_parsers = set(self.get_parser_list()) & self.get_running_topology_names(env) Logger.info('Parsers that need stopped: ' + str(running_parsers)) for parser in running_parsers: Logger.info('Stopping ' + parser) stop_cmd = 'storm kill ' + parser if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) Logger.info('Done stopping parser topologies')
def stop_random_access_indexing_topology(self, env): Logger.info('Stopping ' + self.__random_access_indexing_topology) if self.is_random_access_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) stop_cmd = 'storm kill ' + self.__random_access_indexing_topology Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info("Random Access Indexing topology already stopped") Logger.info('Done stopping random access indexing topologies')
def stop_parser_topologies(self, env): Logger.info('Stopping parsers') running_parsers = set(self.get_parser_aggr_topology_names(self.__params)) & self.get_running_topology_names(env) Logger.info('Parsers that need stopped: ' + str(running_parsers)) for parser in running_parsers: Logger.info('Stopping ' + parser) stop_cmd = 'storm kill ' + parser if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) Logger.info('Done stopping parser topologies')
def stop_indexing_topology(self, env): Logger.info('Stopping ' + self.__indexing_topology) if self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) stop_cmd = 'storm kill ' + self.__indexing_topology Execute(stop_cmd, user=self.__params.metron_user) else: Logger.info("Indexing topology already stopped") Logger.info('Done stopping indexing topologies')
def stop_rest_application(self): """ Stop the REST application """ Logger.info('Stopping REST application') # Get the pid associated with the service pid_file = format("{metron_rest_pid_dir}/{metron_rest_pid}") pid = get_user_call_output.get_user_call_output(format("cat {pid_file}"), user=self.__params.metron_user, is_checked_call=False)[1] process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1") if self.__params.security_enabled: kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) # Politely kill kill_cmd = ('kill', format("{pid}")) Execute(kill_cmd, sudo=True, not_if = format("! ({process_id_exists_command})") ) # Violently kill hard_kill_cmd = ('kill', '-9', format("{pid}")) wait_time = 5 Execute(hard_kill_cmd, not_if = format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"), sudo=True, ignore_failures = True ) try: # check if stopped the process, else fail the task Execute(format("! ({process_id_exists_command})"), tries=20, try_sleep=3, ) except: show_logs(self.__params.metron_log_dir, self.__params.metron_user) raise File(pid_file, action = "delete") Logger.info('Done stopping REST application')
def start_batch_indexing_topology(self, env): Logger.info('Starting ' + self.__batch_indexing_topology) if not self.is_batch_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd_template = """{0}/bin/start_hdfs_topology.sh""" start_cmd = start_cmd_template.format(self.__params.metron_home) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info('Batch Indexing topology already running') Logger.info('Finished starting batch indexing topology')
def start(self, env, upgrade_type=None): from params import params env.set_params(params) self.configure(env) commands = ProfilerCommands(params) if params.security_enabled: metron_security.kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() if params.security_enabled and not commands.is_acl_configured(): commands.init_kafka_acls() commands.set_acl_configured() commands.start_profiler_topology(env)
def create_hbase_tables(self): Logger.info("Creating HBase Tables for indexing") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.hbase_keytab_path, self.__params.hbase_principal_name, execute_user=self.__params.hbase_user) cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" add_update_cmd = cmd.format(self.__params.update_hbase_table, self.__params.update_hbase_cf) Execute(add_update_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user ) Logger.info("Done creating HBase Tables for indexing") self.set_hbase_configured()
def create_hbase_tables(self): Logger.info("Creating HBase Tables for profiler") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.hbase_keytab_path, self.__params.hbase_principal_name, execute_user=self.__params.hbase_user) cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf) Execute(add_table_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=self.__params.hbase_user) Logger.info("Done creating HBase Tables for profiler") self.set_hbase_configured()
def solr_schema_delete(self, env): from params import params env.set_params(params) Logger.info("Deleting Solr schemas") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) commands = IndexingCommands(params) for collection_name in commands.get_solr_schemas(): # delete the schema cmd = format(("export ZOOKEEPER={solr_zookeeper_url};" "export SECURITY_ENABLED={security_enabled};")) cmd += "{0}/bin/delete_collection.sh {1};" Execute(cmd.format(params.metron_home, collection_name), user=self.__params.metron_user)
def start_random_access_indexing_topology(self, env): Logger.info('Starting ' + self.__random_access_indexing_topology) if not self.is_random_access_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh""" if self.__params.ra_indexing_writer == 'Solr': start_cmd_template = """{0}/bin/start_solr_topology.sh""" start_cmd = start_cmd_template.format(self.__params.metron_home) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info('Random Access Indexing topology already running') Logger.info('Finished starting random access indexing topology')
def start_profiler_topology(self, env): Logger.info('Starting ' + self.__profiler_topology) if not self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd_template = """{0}/bin/start_profiler_topology.sh \ -s {1} \ -z {2}""" start_cmd = start_cmd_template.format(self.__params.metron_home, self.__profiler_topology, self.__params.zookeeper_quorum) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info('Profiler topology already running') Logger.info('Finished starting profiler topology')
def check_hbase_table(params, table): """ Validates that an HBase table exists. An exception is raised if the table does not exist. :param params: :param table: The name of the HBase table. """ Logger.info("Checking HBase table '{0}'".format(table)) # if needed kinit as 'hbase' if params.security_enabled: kinit(params.kinit_path_local, params.hbase_keytab_path, params.hbase_principal_name, execute_user=params.hbase_user) template = "echo \"exists '{0}'\" | hbase shell -n | grep 'Table {1} does exist'" cmd = template.format(table, table) err_msg = "Missing HBase table; table={0}".format(table) execute(cmd, user=params.hbase_user, err_msg=err_msg)
def start_profiler_topology(self, env): Logger.info('Starting ' + self.__profiler_topology) if not self.is_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) start_cmd_template = """{0}/bin/start_profiler_topology.sh \ -s {1} \ -z {2}""" Execute(start_cmd_template.format(self.__params.metron_home, self.__profiler_topology, self.__params.zookeeper_quorum), user=self.__params.metron_user) else: Logger.info('Profiler topology already running') Logger.info('Finished starting profiler topology')
def check_hbase_column_family(params, table, column_family): """ Validates that an HBase column family exists. An exception is raised if the column family does not exist. :param params: :param table: The name of the HBase table. :param column_family: The name of the HBase column family. """ Logger.info("Checking column family '{0}:{1}'".format(table, column_family)) # if needed kinit as 'hbase' if params.security_enabled: kinit(params.kinit_path_local, params.hbase_keytab_path, params.hbase_principal_name, execute_user=params.hbase_user) template = "echo \"desc '{0}'\" | hbase shell -n | grep \"NAME => '{1}'\"" cmd = template.format(table, column_family) err_msg = "Missing HBase column family; table={0}, cf={1}".format(table, column_family) execute(cmd, user=params.hbase_user, err_msg=err_msg)
def check_hdfs_file_exists(params, path, user=None): """ Validate that a file exists in HDFS. :param params: :param path: The file path in HDFS. :param user: The user to execute the check under. """ if user is None: user = params.metron_user Logger.info("Checking HDFS; file={0}, user={1}".format(path, user)) # if needed kinit as 'metron' if params.security_enabled: kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) template = "{0}/hdfs dfs -test -f {1}" cmd = template.format(params.hadoop_bin_dir, path) err_msg = "Missing file in HDFS; file={0}".format(path) execute(cmd, user=user, err_msg=err_msg)
def solr_schema_delete(self, env): from params import params env.set_params(params) Logger.info("Deleting Solr schemas") if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.solr_keytab_path, self.__params.solr_principal_name, self.__params.solr_user) commands = IndexingCommands(params) for collection_name in commands.get_solr_schemas(): # delete the schema cmd = format(( "export ZOOKEEPER={solr_zookeeper_url};" "export SECURITY_ENABLED={security_enabled};" )) cmd += "{0}/bin/delete_collection.sh {1};" Execute( cmd.format(params.metron_home, collection_name), user=self.__params.solr_user)
def create_hbase_table(params, table, cf): """ Creates an HBase table, if the table does not currently exist :param params: :param table: The name of the HBase table. :param cf: The column family :param user: The user to execute the command as """ if params.security_enabled: kinit(params.kinit_path_local, params.hbase_keytab_path, params.hbase_principal_name, execute_user=params.hbase_user) cmd = """if [[ $(echo \"exists '{0}'\" | hbase shell | grep 'not exist') ]]; \ then echo \"create '{0}','{1}'\" | hbase shell -n; fi""" add_update_cmd = cmd.format(table, cf) Execute(add_update_cmd, tries=3, try_sleep=5, logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', user=params.hbase_user )
def start(self, env, upgrade_type=None): from params import params env.set_params(params) self.configure(env) commands = EnrichmentCommands(params) if params.security_enabled: metron_security.kinit(params.kinit_path_local, params.metron_keytab_path, params.metron_principal_name, execute_user=params.metron_user) if not commands.is_kafka_configured(): commands.init_kafka_topics() if params.security_enabled and not commands.is_kafka_acl_configured(): commands.init_kafka_acls() if not commands.is_hbase_configured(): commands.create_hbase_tables() if params.security_enabled and not commands.is_hbase_acl_configured(): commands.set_hbase_acls() if not commands.is_maxmind_configured(): commands.init_maxmind() commands.start_enrichment_topology(env)