コード例 #1
0
def pull_config(params):
  Logger.info('Pulling all Metron configs down from ZooKeeper to local file system')
  Logger.info('NOTE - THIS IS OVERWRITING THE LOCAL METRON CONFIG DIR WITH ZOOKEEPER CONTENTS: ' + params.metron_zookeeper_config_path)
  Execute(ambari_format(
      "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PULL --output_dir {metron_zookeeper_config_path} --force"),
      path=ambari_format("{java_home}/bin")
  )
コード例 #2
0
def pull_config(params):
  Logger.info('Pulling all Metron configs down from ZooKeeper to local file system')
  Logger.info('NOTE - THIS IS OVERWRITING THE LOCAL METRON CONFIG DIR WITH ZOOKEEPER CONTENTS: ' + params.metron_zookeeper_config_path)
  Execute(ambari_format(
      "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PULL --output_dir {metron_zookeeper_config_path} --force"),
      path=ambari_format("{java_home}/bin")
  )
コード例 #3
0
    def elasticsearch_template_install(self, env):
        from params import params
        env.set_params(params)

        File(params.bro_index_path,
             mode=0755,
             content=StaticFile('bro_index.template')
             )

        File(params.snort_index_path,
             mode=0755,
             content=StaticFile('snort_index.template')
             )

        File(params.yaf_index_path,
             mode=0755,
             content=StaticFile('yaf_index.template')
             )

        bro_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
        Execute(bro_cmd, logoutput=True)
        snort_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
        Execute(snort_cmd, logoutput=True)
        yaf_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
        Execute(yaf_cmd, logoutput=True)
コード例 #4
0
    def zeppelin_notebook_import(self, env):
        from params import params
        env.set_params(params)
        commands = IndexingCommands(params)

        Logger.info(
            ambari_format(
                'Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'
            ))

        # Check if authentication is configured on Zeppelin server, and fetch details if enabled.
        ses = requests.session()
        ses = commands.get_zeppelin_auth_details(ses,
                                                 params.zeppelin_server_url,
                                                 env)
        for dirName, subdirList, files in os.walk(
                params.metron_config_zeppelin_path):
            for fileName in files:
                if fileName.endswith(".json"):
                    Logger.info("Importing notebook: " + fileName)
                    zeppelin_import_url = ambari_format(
                        'http://{zeppelin_server_url}/api/notebook/import')
                    zeppelin_notebook = {
                        'file': open(os.path.join(dirName, fileName), 'rb')
                    }
                    res = ses.post(zeppelin_import_url,
                                   files=zeppelin_notebook)
                    Logger.info("Result: " + res.text)
コード例 #5
0
    def kibana_dashboard_install(self, env):
      from params import params
      env.set_params(params)

      Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))

      kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template')
      if not os.path.isfile(kibanaTemplate):
        raise IOError(
            errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)

      Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
      template_cmd = ambari_format(
          'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate)
      Execute(template_cmd, logoutput=True)

      kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json')
      if not os.path.isfile(kibanaDashboardLoad):
        raise IOError(
            errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)

      Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)

      kibana_cmd = ambari_format(
          'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad)
      Execute(kibana_cmd, logoutput=True)
コード例 #6
0
    def elasticsearch_template_install(self, env):
        from params import params
        env.set_params(params)

        File(params.bro_index_path,
             mode=0755,
             content=StaticFile('bro_index.template'))

        File(params.snort_index_path,
             mode=0755,
             content=StaticFile('snort_index.template'))

        File(params.yaf_index_path,
             mode=0755,
             content=StaticFile('yaf_index.template'))

        bro_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}'
        )
        Execute(bro_cmd, logoutput=True)
        snort_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}'
        )
        Execute(snort_cmd, logoutput=True)
        yaf_cmd = ambari_format(
            'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}'
        )
        Execute(yaf_cmd, logoutput=True)
コード例 #7
0
ファイル: metron_service.py プロジェクト: zy0001/metron
def init_zk_config(params):
    Logger.info(
        'Loading ALL Metron config into ZooKeeper - this command should ONLY be executed by Ambari on initial install.'
    )
    Execute(ambari_format(
        "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PUSH --input_dir {metron_zookeeper_config_path}"
    ),
            path=ambari_format("{java_home}/bin"))
コード例 #8
0
ファイル: metron_service.py プロジェクト: srini-hdp/metron-1
def metron_knox_topology_setup(params):
    if os.path.exists(params.knox_home):
        File(ambari_format("{knox_home}/conf/topologies/metron.xml"),
             content=Template("metron.xml.j2"),
             owner=params.knox_user,
             group=params.knox_group)
        File(ambari_format("{knox_home}/conf/topologies/metronsso.xml"),
             content=Template("metronsso.xml.j2"),
             owner=params.knox_user,
             group=params.knox_group)
コード例 #9
0
def patch_global_config(params):
  patch_file = "/tmp/metron-global-config-patch.json"
  Logger.info("Setup temporary global config JSON patch (formatting per RFC6902): " + patch_file)
  build_global_config_patch(params, patch_file)

  Logger.info('Patching global config in ZooKeeper')
  Execute(ambari_format(
      "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file),
      path=ambari_format("{java_home}/bin")
  )
コード例 #10
0
    def elasticsearch_template_delete(self, env):
        from params import params
        env.set_params(params)

        bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
        Execute(bro_cmd, logoutput=True)
        snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
        Execute(snort_cmd, logoutput=True)
        yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
        Execute(yaf_cmd, logoutput=True)
コード例 #11
0
def patch_global_config(params):
  patch_file = "/tmp/metron-global-config-patch.json"
  Logger.info("Setup temporary global config JSON patch (formatting per RFC6902): " + patch_file)
  build_global_config_patch(params, patch_file)

  Logger.info('Patching global config in ZooKeeper')
  Execute(ambari_format(
      "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file),
      path=ambari_format("{java_home}/bin")
  )
コード例 #12
0
    def zeppelin_notebook_import(self, env):
        from params import params
        env.set_params(params)

        Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
        for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
            for fileName in files:
                if fileName.endswith(".json"):
                    zeppelin_cmd = ambari_format(
                        'curl -s -XPOST http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, fileName) + '"')
                    Execute(zeppelin_cmd, logoutput=True)
コード例 #13
0
def metron_knox_topology_setup(params):
    if os.path.exists(params.knox_home):
        File(ambari_format("{knox_home}/conf/topologies/metron.xml"),
             content=Template("metron.xml.j2"),
             owner=params.knox_user,
             group=params.knox_group
             )
        File(ambari_format("{knox_home}/conf/topologies/metronsso.xml"),
             content=Template("metronsso.xml.j2"),
             owner=params.knox_user,
             group=params.knox_group
             )
コード例 #14
0
ファイル: indexing_commands.py プロジェクト: ajj31/sizak
    def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
        """
        With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password
        for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details).

        This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the
        session connection details to be used for importing Zeppelin notebooks.
        :param ses: Session handle
        :param zeppelin_server_url: Zeppelin Server URL
        :return: ses
        """
        from params import params
        env.set_params(params)

        # Check if authentication is enabled on the Zeppelin server
        try:
            ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))

            # Establish connection if authentication is enabled
            try:
                Logger.info(
                    "Shiro authentication is found to be enabled on the Zeppelin server."
                )
                # Read the Shiro admin user credentials from Zeppelin config in Ambari
                seen_users = False
                username = None
                password = None
                if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content,
                             re.MULTILINE):
                    seen_users = True
                    tokens = re.search(r'^admin\ =.*',
                                       params.zeppelin_shiro_ini_content,
                                       re.MULTILINE).group()
                    userpassword = tokens.split(',')[0].strip()
                    username = userpassword.split('=')[0].strip()
                    password = userpassword.split('=')[1].strip()
                else:
                    Logger.error(
                        "ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail."
                    )

                zeppelin_payload = {'userName': username, 'password': password}
                ses.post(
                    ambari_format('http://{zeppelin_server_url}/api/login'),
                    data=zeppelin_payload)
            except:
                pass

        # If authentication is not enabled, fall back to default method of imporing notebooks
        except requests.exceptions.RequestException:
            ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))

        return ses
コード例 #15
0
    def elasticsearch_template_delete(self, env):
        from params import params
        env.set_params(params)

        bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
        Execute(bro_cmd, logoutput=True)
        snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
        Execute(snort_cmd, logoutput=True)
        yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
        Execute(yaf_cmd, logoutput=True)
        error_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/error_index*"')
        Execute(error_cmd, logoutput=True)
コード例 #16
0
def storm_security_setup(params):
    if params.security_enabled:
        # I don't think there's an Ambari way to get a user's local home dir , so have Python perform tilde expansion.
        # Ambari's Directory doesn't do tilde expansion.
        metron_storm_dir_tilde = '~' + params.metron_user + '/.storm'
        metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde)

        Directory(params.metron_home,
                  mode=0755,
                  owner=params.metron_user,
                  group=params.metron_group,
                  create_parents=True)

        Directory(metron_storm_dir,
                  mode=0755,
                  owner=params.metron_user,
                  group=params.metron_group)

        File(ambari_format('{client_jaas_path}'),
             content=InlineTemplate(params.metron_client_jaas_conf_template),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755)

        File(metron_storm_dir + '/storm.yaml',
             content=Template('storm.yaml.j2'),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755)

        File(metron_storm_dir + '/storm.config',
             content=Template('storm.config.j2'),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755)
コード例 #17
0
def get_running_topologies(params):
    Logger.info('Getting Running Storm Topologies from Storm REST Server')

    Logger.info('Security enabled? ' + str(params.security_enabled))

    # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets.
    # This is becuase we need to run a command with a return as the metron user. Sigh
    negotiate = '--negotiate -u : ' if params.security_enabled else ''
    cmd = ambari_format('curl --max-time 3 ' + negotiate +
                        '{storm_rest_addr}/api/v1/topology/summary')

    if params.security_enabled:
        kinit(params.kinit_path_local,
              params.metron_keytab_path,
              params.metron_principal_name,
              execute_user=params.metron_user)

    Logger.info('Running cmd: ' + cmd)
    return_code, stdout, stderr = get_user_call_output(cmd,
                                                       user=params.metron_user,
                                                       is_checked_call=False)

    if (return_code != 0):
        return {}

    try:
        stormjson = json.loads(stdout)
    except ValueError, e:
        Logger.info('Stdout: ' + str(stdout))
        Logger.info('Stderr: ' + str(stderr))
        Logger.exception(str(e))
        return {}
コード例 #18
0
def get_running_topologies(params):
  Logger.info('Getting Running Storm Topologies from Storm REST Server')
  Logger.info('Security enabled? ' + str(params.security_enabled))

  # Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets.
  # This is becuase we need to run a command with a return as the metron user. Sigh
  negotiate = '--negotiate -u : ' if params.security_enabled else ''
  cmd = ambari_format(
    'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary')

  if params.security_enabled:
    kinit(params.kinit_path_local,
          params.metron_keytab_path,
          params.metron_principal_name,
          execute_user=params.metron_user)

  Logger.info('Running cmd: ' + cmd)
  return_code, stdout, stderr = get_user_call_output(cmd,
                                                     user=params.metron_user,
                                                     is_checked_call=False)

  if (return_code != 0):
    return {}

  try:
    stormjson = json.loads(stdout)
  except ValueError, e:
    Logger.info('Stdout: ' + str(stdout))
    Logger.info('Stderr: ' + str(stderr))
    Logger.exception(str(e))
    return {}
コード例 #19
0
    def zeppelin_notebook_import(self, env):
        from params import params
        env.set_params(params)
        commands = IndexingCommands(params)

        Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))

        # Check if authentication is configured on Zeppelin server, and fetch details if enabled.
        ses = requests.session()
        ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env)
        for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
            for fileName in files:
                if fileName.endswith(".json"):
                    Logger.info("Importing notebook: " + fileName)
                    zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import')
                    zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')}
                    res = ses.post(zeppelin_import_url, files=zeppelin_notebook)
                    Logger.info("Result: " + res.text)
コード例 #20
0
    def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
        """
        With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password
        for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details).

        This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the
        session connection details to be used for importing Zeppelin notebooks.
        :param ses: Session handle
        :param zeppelin_server_url: Zeppelin Server URL
        :return: ses
        """
        from params import params
        env.set_params(params)

        # Check if authentication is enabled on the Zeppelin server
        try:
            ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))

            # Establish connection if authentication is enabled
            try:
                Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
                # Read the Shiro admin user credentials from Zeppelin config in Ambari
                seen_users = False
                username = None
                password = None
                if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
                    seen_users = True
                    tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
                    userpassword = tokens.split(',')[0].strip()
                    username = userpassword.split('=')[0].strip()
                    password = userpassword.split('=')[1].strip()
                else:
                    Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")

                zeppelin_payload = {'userName': username, 'password' : password}
                ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload)
            except:
                pass

        # If authentication is not enabled, fall back to default method of imporing notebooks
        except requests.exceptions.RequestException:
            ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))

        return ses
コード例 #21
0
def load_global_config(params):
    Logger.info('Create Metron Local Config Directory')
    Logger.info("Configure Metron global.json")

    directories = [params.metron_zookeeper_config_path]
    Directory(directories,
              mode=0755,
              owner=params.metron_user,
              group=params.metron_group)

    File(ambari_format("{metron_zookeeper_config_path}/global.json"),
         content=Template("global.json.j2"),
         owner=params.metron_user,
         group=params.metron_group)

    init_config()
コード例 #22
0
def get_running_topologies():
    Logger.info('Getting Running Storm Topologies from Storm REST Server')

    cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    (stdout, stderr) = proc.communicate()

    try:
        stormjson = json.loads(stdout)
    except ValueError:
        return {}

    topologiesDict = {}

    for topology in stormjson['topologies']:
        topologiesDict[topology['name']] = topology['status']

    Logger.info("Topologies: " + str(topologiesDict))
    return topologiesDict
コード例 #23
0
def storm_security_setup(params):
    if params.security_enabled:
        # I don't think there's an Ambari way to get a user's local home dir , so have Python perform tilde expansion.
        # Ambari's Directory doesn't do tilde expansion.
        metron_storm_dir_tilde = '~' + params.metron_user + '/.storm'
        metron_storm_dir = os.path.expanduser(metron_storm_dir_tilde)


        Directory(params.metron_home,
                  mode=0755,
                  owner=params.metron_user,
                  group=params.metron_group,
                  create_parents=True
                  )

        Directory(metron_storm_dir,
                  mode=0755,
                  owner=params.metron_user,
                  group=params.metron_group
                  )

        File(ambari_format('{client_jaas_path}'),
             content=Template('client_jaas.conf.j2'),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755
             )

        File(metron_storm_dir + '/storm.yaml',
             content=Template('storm.yaml.j2'),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755
             )

        File(metron_storm_dir + '/storm.config',
             content=Template('storm.config.j2'),
             owner=params.metron_user,
             group=params.metron_group,
             mode=0755
             )
コード例 #24
0
    def zeppelin_notebook_import(self, env):
        from params import params
        env.set_params(params)
        metron_service.check_indexer_parameters()

        commands = IndexingCommands(params)
        Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))

        # Check if authentication is configured on Zeppelin server, and fetch details if enabled.
        session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env)
        for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
            for fileName in files:
                if fileName.endswith(".json"):
                    Logger.info("Importing notebook: " + fileName)
                    zeppelin_notebook = os.path.join(dirName, fileName)
                    zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\''
                    zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook)
                    return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user)
                    Logger.info("Status of importing notebook: " + import_result)
                    if return_code != 0:
                        Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
コード例 #25
0
def init_config():
    Logger.info('Loading config into ZooKeeper')
    Execute(ambari_format(
        "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
        path=ambari_format("{java_home}/bin")
    )
コード例 #26
0
ファイル: metron_service.py プロジェクト: zy0001/metron
def build_global_config_patch(params, patch_file):
    """
  Build the file used to patch the global configuration.
  See RFC 6902 at https://tools.ietf.org/html/rfc6902

  :param params:
  :param patch_file: The path where the patch file will be created.
  """
    if params.ra_indexing_writer == 'Solr':
        indexing_patches = solr_global_config_patches()
    else:
        indexing_patches = elasticsearch_global_config_patches()
    other_patches = """
    {
        "op": "add",
        "path": "/profiler.client.period.duration",
        "value": "{{profiler_period_duration}}"
    },
    {
        "op": "add",
        "path": "/profiler.client.period.duration.units",
        "value": "{{profiler_period_units}}"
    },
    {
        "op": "add",
        "path": "/parser.error.topic",
        "value": "{{parser_error_topic}}"
    },
    {
        "op": "add",
        "path": "/update.hbase.table",
        "value": "{{update_hbase_table}}"
    },
    {
        "op": "add",
        "path": "/update.hbase.cf",
        "value": "{{update_hbase_cf}}"
    },
    {
        "op": "add",
        "path": "/user.settings.hbase.table",
        "value": "{{user_settings_hbase_table}}"
    },
    {
        "op": "add",
        "path": "/user.settings.hbase.cf",
        "value": "{{user_settings_hbase_cf}}"
    },
    {
        "op": "add",
        "path": "/bootstrap.servers",
        "value": "{{kafka_brokers}}"
    },
    {
        "op": "add",
        "path": "/source.type.field",
        "value": "{{source_type_field}}"
    },
    {
        "op": "add",
        "path": "/threat.triage.score.field",
        "value": "{{threat_triage_score_field}}"
    },
    {
        "op": "add",
        "path": "/enrichment.writer.batchSize",
        "value": "{{enrichment_kafka_writer_batch_size}}"
    },
    {
        "op": "add",
        "path": "/enrichment.writer.batchTimeout",
        "value": "{{enrichment_kafka_writer_batch_timeout}}"
    },
    {
        "op": "add",
        "path": "/profiler.writer.batchSize",
        "value": "{{profiler_kafka_writer_batch_size}}"
    },
    {
        "op": "add",
        "path": "/profiler.writer.batchTimeout",
        "value": "{{profiler_kafka_writer_batch_timeout}}"
    }
  """
    patch_template = ambari_format("""
  [
    {indexing_patches},
    {other_patches}
  ]
  """)
    File(patch_file,
         content=InlineTemplate(patch_template),
         owner=params.metron_user,
         group=params.metron_group)
コード例 #27
0
def init_config():
    Logger.info('Loading config into ZooKeeper')
    Execute(ambari_format(
        "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"
    ),
            path=ambari_format("{java_home}/bin"))
コード例 #28
0
def build_global_config_patch(params, patch_file):
  """
  Build the file used to patch the global configuration.
  See RFC 6902 at https://tools.ietf.org/html/rfc6902

  :param params:
  :param patch_file: The path where the patch file will be created.
  """
  if params.ra_indexing_writer == 'Solr':
      indexing_patches = solr_global_config_patches()
  else:
      indexing_patches = elasticsearch_global_config_patches()
  other_patches = """
    {
        "op": "add",
        "path": "/profiler.client.period.duration",
        "value": "{{profiler_period_duration}}"
    },
    {
        "op": "add",
        "path": "/profiler.client.period.duration.units",
        "value": "{{profiler_period_units}}"
    },
    {
        "op": "add",
        "path": "/parser.error.topic",
        "value": "{{parser_error_topic}}"
    },
    {
        "op": "add",
        "path": "/update.hbase.table",
        "value": "{{update_hbase_table}}"
    },
    {
        "op": "add",
        "path": "/update.hbase.cf",
        "value": "{{update_hbase_cf}}"
    },
    {
        "op": "add",
        "path": "/user.settings.hbase.table",
        "value": "{{user_settings_hbase_table}}"
    },
    {
        "op": "add",
        "path": "/user.settings.hbase.cf",
        "value": "{{user_settings_hbase_cf}}"
    },
    {
        "op": "add",
        "path": "/bootstrap.servers",
        "value": "{{kafka_brokers}}"
    },
    {
        "op": "add",
        "path": "/source.type.field",
        "value": "{{source_type_field}}"
    },
    {
        "op": "add",
        "path": "/threat.triage.score.field",
        "value": "{{threat_triage_score_field}}"
    },
    {
        "op": "add",
        "path": "/enrichment.writer.batchSize",
        "value": "{{enrichment_kafka_writer_batch_size}}"
    },
    {
        "op": "add",
        "path": "/enrichment.writer.batchTimeout",
        "value": "{{enrichment_kafka_writer_batch_timeout}}"
    },
    {
        "op": "add",
        "path": "/profiler.writer.batchSize",
        "value": "{{profiler_kafka_writer_batch_size}}"
    },
    {
        "op": "add",
        "path": "/profiler.writer.batchTimeout",
        "value": "{{profiler_kafka_writer_batch_timeout}}"
    }
  """
  patch_template = ambari_format(
  """
  [
    {indexing_patches},
    {other_patches}
  ]
  """)
  File(patch_file,
       content=InlineTemplate(patch_template),
       owner=params.metron_user,
       group=params.metron_group)
コード例 #29
0
def init_zk_config(params):
  Logger.info('Loading ALL Metron config into ZooKeeper - this command should ONLY be executed by Ambari on initial install.')
  Execute(ambari_format(
      "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PUSH --input_dir {metron_zookeeper_config_path}"),
      path=ambari_format("{java_home}/bin")
  )