Пример #1
0
    def start(self, env, upgrade_type=None):
        import params
        env.set_params(params)
        self.configure(env, upgrade_type=upgrade_type)

        if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
            solr_cloud_util.setup_solr_client(params.config,
                                              custom_log4j=params.custom_log4j)
            setup_ranger_audit_solr()

        ranger_service('ranger_admin')
Пример #2
0
  def start(self, env, upgrade_type=None):
    import params
    env.set_params(params)

    # setup db only if in case stack version is > 2.6
    self.configure(env, upgrade_type=upgrade_type, setup_db=params.stack_supports_ranger_setup_db_on_start)

    if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
      solr_cloud_util.setup_solr_client(params.config, custom_log4j = params.custom_log4j)
      setup_ranger_audit_solr()

    ranger_service('ranger_admin')
Пример #3
0
    def start(self, env, upgrade_type=None):
        import params
        env.set_params(params)
        self.install_ranger()
        Execute(
            "chown -R %s:%s %s" %
            (params.unix_user, params.unix_group, params.install_dir_admin))
        # setup db only if in case HDP version is > 2.6
        self.configure(env,
                       upgrade_type=upgrade_type,
                       setup_db=params.stack_supports_ranger_setup_db_on_start)

        if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
            solr_cloud_util.setup_solr_client(params.config,
                                              custom_log4j=params.custom_log4j)
            setup_ranger_audit_solr()

        ranger_service('ranger_admin')
Пример #4
0
def setup_infra_solr(name=None):
    import params

    if name == 'server':
        Directory([
            params.infra_solr_log_dir, params.infra_solr_piddir,
            params.infra_solr_datadir, params.infra_solr_data_resources_dir
        ],
                  mode=0755,
                  cd_access='a',
                  create_parents=True,
                  owner=params.infra_solr_user,
                  group=params.user_group)

        Directory([params.solr_dir, params.infra_solr_conf],
                  mode=0755,
                  cd_access='a',
                  owner=params.infra_solr_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)

        File(params.infra_solr_log,
             mode=0644,
             owner=params.infra_solr_user,
             group=params.user_group,
             content='')

        File(format("{infra_solr_conf}/infra-solr-env.sh"),
             content=InlineTemplate(params.solr_env_content),
             mode=0755,
             owner=params.infra_solr_user,
             group=params.user_group)

        File(format("{infra_solr_datadir}/solr.xml"),
             content=InlineTemplate(params.solr_xml_content),
             owner=params.infra_solr_user,
             group=params.user_group)

        File(format("{infra_solr_conf}/log4j2.xml"),
             content=InlineTemplate(params.solr_log4j_content),
             owner=params.infra_solr_user,
             group=params.user_group)

        custom_security_json_location = format(
            "{infra_solr_conf}/custom-security.json")
        File(custom_security_json_location,
             content=InlineTemplate(params.infra_solr_security_json_content),
             owner=params.infra_solr_user,
             group=params.user_group,
             mode=0640)

        if params.security_enabled:
            File(format("{infra_solr_jaas_file}"),
                 content=Template("infra_solr_jaas.conf.j2"),
                 owner=params.infra_solr_user)

            File(format("{infra_solr_conf}/security.json"),
                 content=Template("infra-solr-security.json.j2"),
                 owner=params.infra_solr_user,
                 group=params.user_group,
                 mode=0640)
        if os.path.exists(params.limits_conf_dir):
            File(os.path.join(params.limits_conf_dir, 'infra-solr.conf'),
                 owner='root',
                 group='root',
                 mode=0644,
                 content=Template("infra-solr.conf.j2"))

    elif name == 'client':
        solr_cloud_util.setup_solr_client(params.config)

    else:
        raise Fail('Nor client or server were selected to install.')
Пример #5
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0755,
             content=InlineTemplate(params.metadata_env_content))

        if not is_empty(params.atlas_admin_username) and not is_empty(
                params.atlas_admin_password):
            psswd_output = hashlib.sha256(
                params.atlas_admin_password).hexdigest()
            ModifyPropertiesFile(
                format("{conf_dir}/users-credentials.properties"),
                properties={
                    format('{atlas_admin_username}'):
                    format('ROLE_ADMIN::{psswd_output}')
                },
                owner=params.metadata_user)

        files_to_chown = [
            format("{conf_dir}/policy-store.txt"),
            format("{conf_dir}/users-credentials.properties")
        ]
        for file in files_to_chown:
            if os.path.exists(file):
                Execute(
                    ('chown', format('{metadata_user}:{user_group}'), file),
                    sudo=True)
                Execute(('chmod', '644', file), sudo=True)

        if params.metadata_solrconfig_content:
            File(format("{conf_dir}/solr/solrconfig.xml"),
                 mode=0644,
                 owner=params.metadata_user,
                 group=params.user_group,
                 content=InlineTemplate(params.metadata_solrconfig_content))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0600,
                   owner=params.metadata_user,
                   group=params.user_group)

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)

    if type == 'server' and params.search_backend_solr and params.has_infra_solr:
        solr_cloud_util.setup_solr_client(params.config)
        check_znode()
        jaasFile = params.atlas_jaas_file if params.security_enabled else None
        upload_conf_set('atlas_configs', jaasFile)

        if params.security_enabled:  # update permissions before creating the collections
            solr_cloud_util.add_solr_roles(
                params.config,
                roles=[
                    params.infra_solr_role_atlas,
                    params.infra_solr_role_ranger_audit,
                    params.infra_solr_role_dev
                ],
                new_service_principals=[params.atlas_jaas_principal])

        create_collection('vertex_index', 'atlas_configs', jaasFile)
        create_collection('edge_index', 'atlas_configs', jaasFile)
        create_collection('fulltext_index', 'atlas_configs', jaasFile)

        if params.security_enabled:
            secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
                         jaasFile)
            secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
                         jaasFile)
            secure_znode(format('{infra_solr_znode}/collections/edge_index'),
                         jaasFile)
            secure_znode(
                format('{infra_solr_znode}/collections/fulltext_index'),
                jaasFile)

    File(params.atlas_hbase_setup,
         group=params.user_group,
         owner=params.hbase_user,
         content=Template("atlas_hbase_setup.rb.j2"))

    is_atlas_upgrade_support = check_stack_feature(
        StackFeature.ATLAS_UPGRADE_SUPPORT,
        get_stack_feature_version(params.config))

    if is_atlas_upgrade_support and params.security_enabled:

        File(params.atlas_kafka_setup,
             group=params.user_group,
             owner=params.kafka_user,
             content=Template("atlas_kafka_acl.sh.j2"))

        #  files required only in case if kafka broker is not present on the host as configured component
        if not params.host_with_kafka:
            File(format("{kafka_conf_dir}/kafka-env.sh"),
                 owner=params.kafka_user,
                 content=InlineTemplate(params.kafka_env_sh_template))

            File(format("{kafka_conf_dir}/kafka_jaas.conf"),
                 group=params.user_group,
                 owner=params.kafka_user,
                 content=Template("kafka_jaas.conf.j2"))

    if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
            params.namenode_host) > 1:
        XmlConfig(
            "hdfs-site.xml",
            conf_dir=params.conf_dir,
            configurations=params.config['configurations']['hdfs-site'],
            configuration_attributes=params.config['configurationAttributes']
            ['hdfs-site'],
            owner=params.metadata_user,
            group=params.user_group,
            mode=0644)
    else:
        File(format('{conf_dir}/hdfs-site.xml'), action="delete")
    '''
    Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for
    authentication and authorization process. Earlier the core-site.xml was available in
    Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards,
    core-site.xml is no more available in Hbase conf directory. Hence need to create
    core-site.xml in Atlas conf directory.
    '''
    if params.stack_supports_atlas_core_site and params.has_namenode:
        XmlConfig(
            "core-site.xml",
            conf_dir=params.conf_dir,
            configurations=params.config['configurations']['core-site'],
            configuration_attributes=params.config['configurationAttributes']
            ['core-site'],
            owner=params.metadata_user,
            group=params.user_group,
            mode=0644)

    Directory(
        format('{metadata_home}/'),
        owner=params.metadata_user,
        group=params.user_group,
        recursive_ownership=True,
    )
Пример #6
0
def setup_infra_solr(name=None):
    import params

    if name == 'server':
        Directory([
            params.infra_solr_log_dir, params.infra_solr_piddir,
            params.infra_solr_datadir, params.infra_solr_data_resources_dir
        ],
                  mode=0755,
                  cd_access='a',
                  create_parents=True,
                  owner=params.infra_solr_user,
                  group=params.user_group)

        Directory([params.solr_dir, params.infra_solr_conf],
                  mode=0755,
                  cd_access='a',
                  owner=params.infra_solr_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)

        File(params.infra_solr_log,
             mode=0644,
             owner=params.infra_solr_user,
             group=params.user_group,
             content='')

        File(format("{infra_solr_conf}/infra-solr-env.sh"),
             content=InlineTemplate(params.solr_env_content),
             mode=0755,
             owner=params.infra_solr_user,
             group=params.user_group)

        File(format("{infra_solr_datadir}/solr.xml"),
             content=InlineTemplate(params.solr_xml_content),
             owner=params.infra_solr_user,
             group=params.user_group)

        File(format("{infra_solr_conf}/log4j.properties"),
             content=InlineTemplate(params.solr_log4j_content),
             owner=params.infra_solr_user,
             group=params.user_group)

        custom_security_json_location = format(
            "{infra_solr_conf}/custom-security.json")
        File(custom_security_json_location,
             content=InlineTemplate(params.infra_solr_security_json_content),
             owner=params.infra_solr_user,
             group=params.user_group,
             mode=0640)

        jaas_file = params.infra_solr_jaas_file if params.security_enabled else None
        url_scheme = 'https' if params.infra_solr_ssl_enabled else 'http'

        create_ambari_solr_znode()

        security_json_file_location = custom_security_json_location \
          if params.infra_solr_security_json_content and str(params.infra_solr_security_json_content).strip() \
          else format("{infra_solr_conf}/security.json") # security.json file to upload

        if params.security_enabled:
            File(format("{infra_solr_jaas_file}"),
                 content=Template("infra_solr_jaas.conf.j2"),
                 owner=params.infra_solr_user)

            File(format("{infra_solr_conf}/security.json"),
                 content=Template("infra-solr-security.json.j2"),
                 owner=params.infra_solr_user,
                 group=params.user_group,
                 mode=0640)

        solr_cloud_util.set_cluster_prop(
            zookeeper_quorum=params.zookeeper_quorum,
            solr_znode=params.infra_solr_znode,
            java64_home=params.java64_home,
            prop_name="urlScheme",
            prop_value=url_scheme,
            jaas_file=jaas_file)

        solr_cloud_util.setup_kerberos_plugin(
            zookeeper_quorum=params.zookeeper_quorum,
            solr_znode=params.infra_solr_znode,
            jaas_file=jaas_file,
            java64_home=params.java64_home,
            secure=params.security_enabled,
            security_json_location=security_json_file_location)

        if params.security_enabled:
            solr_cloud_util.secure_solr_znode(
                zookeeper_quorum=params.zookeeper_quorum,
                solr_znode=params.infra_solr_znode,
                jaas_file=jaas_file,
                java64_home=params.java64_home,
                sasl_users_str=params.infra_solr_sasl_user)

    elif name == 'client':
        solr_cloud_util.setup_solr_client(params.config)

    else:
        raise Fail('Nor client or server were selected to install.')
Пример #7
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0755,
             content=InlineTemplate(params.metadata_env_content))
        if params.metadata_solrconfig_content:
            File(format("{conf_dir}/solr/solrconfig.xml"),
                 mode=0644,
                 owner=params.metadata_user,
                 group=params.user_group,
                 content=InlineTemplate(params.metadata_solrconfig_content))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0644,
                   owner=params.metadata_user,
                   group=params.user_group)

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)

    if type == 'server' and params.search_backend_solr and params.has_infra_solr:
        solr_cloud_util.setup_solr_client(params.config)
        check_znode()
        jaasFile = params.atlas_jaas_file if params.security_enabled else None
        upload_conf_set('atlas_configs', jaasFile)

        create_collection('vertex_index', 'atlas_configs', jaasFile)
        create_collection('edge_index', 'atlas_configs', jaasFile)
        create_collection('fulltext_index', 'atlas_configs', jaasFile)

    File(params.atlas_hbase_setup,
         group=params.user_group,
         owner=params.hbase_user,
         content=Template("atlas_hbase_setup.rb.j2"))

    if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) and\
      params.security_enabled and not params.host_with_kafka:

        File(params.atlas_kafka_setup,
             group=params.user_group,
             owner=params.kafka_user,
             content=Template("atlas_kafka_acl.sh.j2"))

        File(format("{kafka_conf_dir}/kafka-env.sh"),
             owner=params.kafka_user,
             content=InlineTemplate(params.kafka_env_sh_template))

        File(format("{kafka_conf_dir}/kafka_jaas.conf"),
             group=params.user_group,
             owner=params.kafka_user,
             content=Template("kafka_jaas.conf.j2"))
Пример #8
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0755,
             content=InlineTemplate(params.metadata_env_content))

        files_to_chown = [
            format("{conf_dir}/policy-store.txt"),
            format("{conf_dir}/users-credentials.properties")
        ]
        for file in files_to_chown:
            if os.path.exists(file):
                Execute(
                    ('chown', format('{metadata_user}:{user_group}'), file),
                    sudo=True)
                Execute(('chmod', '644', file), sudo=True)

        if params.metadata_solrconfig_content:
            File(format("{conf_dir}/solr/solrconfig.xml"),
                 mode=0644,
                 owner=params.metadata_user,
                 group=params.user_group,
                 content=InlineTemplate(params.metadata_solrconfig_content))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0644,
                   owner=params.metadata_user,
                   group=params.user_group)

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)

    if type == 'server' and params.search_backend_solr and params.has_infra_solr:
        solr_cloud_util.setup_solr_client(params.config)
        check_znode()
        jaasFile = params.atlas_jaas_file if params.security_enabled else None
        upload_conf_set('atlas_configs', jaasFile)

        create_collection('vertex_index', 'atlas_configs', jaasFile)
        create_collection('edge_index', 'atlas_configs', jaasFile)
        create_collection('fulltext_index', 'atlas_configs', jaasFile)

    File(params.atlas_hbase_setup,
         group=params.user_group,
         owner=params.hbase_user,
         content=Template("atlas_hbase_setup.rb.j2"))

    is_atlas_upgrade_support = check_stack_feature(
        StackFeature.ATLAS_UPGRADE_SUPPORT,
        get_stack_feature_version(params.config))

    if is_atlas_upgrade_support and params.security_enabled:

        File(params.atlas_kafka_setup,
             group=params.user_group,
             owner=params.kafka_user,
             content=Template("atlas_kafka_acl.sh.j2"))

        #  files required only in case if kafka broker is not present on the host as configured component
        if not params.host_with_kafka:
            File(format("{kafka_conf_dir}/kafka-env.sh"),
                 owner=params.kafka_user,
                 content=InlineTemplate(params.kafka_env_sh_template))

            File(format("{kafka_conf_dir}/kafka_jaas.conf"),
                 group=params.user_group,
                 owner=params.kafka_user,
                 content=Template("kafka_jaas.conf.j2"))

    if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
            params.namenode_host) > 1:
        XmlConfig(
            "hdfs-site.xml",
            conf_dir=params.conf_dir,
            configurations=params.config['configurations']['hdfs-site'],
            configuration_attributes=params.config['configuration_attributes']
            ['hdfs-site'],
            owner=params.metadata_user,
            group=params.user_group,
            mode=0644)
    else:
        File(format('{conf_dir}/hdfs-site.xml'), action="delete")
Пример #9
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0755,
             content=InlineTemplate(params.metadata_env_content))
        File(format("{conf_dir}/solr/solrconfig.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_solrconfig_content))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0644,
                   owner=params.metadata_user,
                   group=params.user_group)

    if type == 'server' and params.search_backend_solr and params.has_logsearch_solr:
        solr_cloud_util.setup_solr_client(params.config)

        random_num = random.random()

        upload_conf_set('basic_configs', random_num)

        create_collection('vertex_index', 'basic_configs')
        create_collection('edge_index', 'basic_configs')
        create_collection('fulltext_index', 'basic_configs')

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)
Пример #10
0
def setup_logsearch_solr(name=None):
    import params

    if name == 'server':
        Directory([
            params.logsearch_solr_log_dir, params.logsearch_solr_piddir,
            params.logsearch_solr_datadir,
            params.logsearch_solr_data_resources_dir
        ],
                  mode=0755,
                  cd_access='a',
                  create_parents=True,
                  owner=params.logsearch_solr_user,
                  group=params.user_group)

        Directory([params.solr_dir, params.logsearch_solr_conf],
                  mode=0755,
                  cd_access='a',
                  owner=params.logsearch_solr_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)

        File(params.logsearch_solr_log,
             mode=0644,
             owner=params.logsearch_solr_user,
             group=params.user_group,
             content='')

        File(format("{logsearch_solr_conf}/logsearch-solr-env.sh"),
             content=InlineTemplate(params.solr_env_content),
             mode=0755,
             owner=params.logsearch_solr_user,
             group=params.user_group)

        File(format("{logsearch_solr_datadir}/solr.xml"),
             content=InlineTemplate(params.solr_xml_content),
             owner=params.logsearch_solr_user,
             group=params.user_group)

        File(format("{logsearch_solr_conf}/log4j.properties"),
             content=InlineTemplate(params.solr_log4j_content),
             owner=params.logsearch_solr_user,
             group=params.user_group)

        File(format("{logsearch_solr_datadir}/zoo.cfg"),
             content=Template("zoo.cfg.j2"),
             owner=params.logsearch_solr_user,
             group=params.user_group)
        zk_cli_prefix = format(
            'export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}'
        )
        create_ambari_solr_znode(zk_cli_prefix)

        url_scheme = 'https' if params.logsearch_solr_ssl_enabled else 'http'
        Execute(format(
            '{zk_cli_prefix}{logsearch_solr_znode} -cmd clusterprop -name urlScheme -val {url_scheme}'
        ),
                user=params.logsearch_solr_user)

        if params.security_enabled:
            File(format("{logsearch_solr_jaas_file}"),
                 content=Template("logsearch_solr_jaas.conf.j2"),
                 owner=params.logsearch_solr_user)
            security_content = '\'{"authentication":{"class": "org.apache.solr.security.KerberosPlugin"}}\''
        else:
            security_content = '\'{}\''
        Execute(format(
            '{zk_cli_prefix} -cmd put {logsearch_solr_znode}/security.json ') +
                security_content,
                user=params.logsearch_solr_user)

    elif name == 'client':
        solr_cloud_util.setup_solr_client(params.config)
        if params.security_enabled:
            File(format("{solr_client_dir}/logsearch_solr_client_jaas.conf"),
                 content=Template("logsearch_solr_jaas.conf.j2"),
                 owner=params.logsearch_solr_user)

    else:
        raise Fail('Nor client or server were selected to install.')
Пример #11
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True,
                  recursive_ownership=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0644,
             content=InlineTemplate(params.metadata_env_content))

        if not is_empty(params.atlas_admin_username) and not is_empty(
                params.atlas_admin_password):
            psswd_output = hashlib.sha256(
                params.atlas_admin_password).hexdigest()
            ModifyPropertiesFile(
                format("{conf_dir}/users-credentials.properties"),
                properties={
                    format('{atlas_admin_username}'):
                    format('ROLE_ADMIN::{psswd_output}')
                },
                owner=params.metadata_user)

        files_to_chown = [
            format("{conf_dir}/atlas-simple-authz-policy.json"),
            format("{conf_dir}/users-credentials.properties")
        ]
        for file in files_to_chown:
            if os.path.exists(file):
                Execute(
                    ('chown', format('{metadata_user}:{user_group}'), file),
                    sudo=True)
                Execute(('chmod', '640', file), sudo=True)

        if params.metadata_solrconfig_content:
            File(format("{conf_dir}/solr/solrconfig.xml"),
                 mode=0644,
                 owner=params.metadata_user,
                 group=params.user_group,
                 content=InlineTemplate(params.metadata_solrconfig_content))

        generate_logfeeder_input_config(
            'atlas',
            Template("input.config-atlas.json.j2", extra_imports=[default]))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0600,
                   owner=params.metadata_user,
                   group=params.user_group)

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)

    if type == 'server' and params.search_backend_solr and params.has_infra_solr:
        solr_cloud_util.setup_solr_client(params.config)
        check_znode()
        jaasFile = params.atlas_jaas_file if params.security_enabled else None
        upload_conf_set('atlas_configs', jaasFile)

        if params.security_enabled:  # update permissions before creating the collections
            solr_cloud_util.add_solr_roles(
                params.config,
                roles=[
                    params.infra_solr_role_atlas,
                    params.infra_solr_role_ranger_audit,
                    params.infra_solr_role_dev
                ],
                new_service_principals=[params.atlas_jaas_principal])

        create_collection('vertex_index', 'atlas_configs', jaasFile)
        create_collection('edge_index', 'atlas_configs', jaasFile)
        create_collection('fulltext_index', 'atlas_configs', jaasFile)

        if params.security_enabled:
            secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
                         jaasFile)
            secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
                         jaasFile)
            secure_znode(format('{infra_solr_znode}/collections/edge_index'),
                         jaasFile)
            secure_znode(
                format('{infra_solr_znode}/collections/fulltext_index'),
                jaasFile)

    File(params.atlas_hbase_setup,
         group=params.user_group,
         owner=params.hbase_user,
         content=Template("atlas_hbase_setup.rb.j2"))

    is_atlas_upgrade_support = True

    if is_atlas_upgrade_support and params.security_enabled:

        File(params.atlas_kafka_setup,
             group=params.user_group,
             owner=params.kafka_user,
             content=Template("atlas_kafka_acl.sh.j2"))

        #  files required only in case if kafka broker is not present on the host as configured component
        if not params.host_with_kafka:
            File(format("{kafka_conf_dir}/kafka-env.sh"),
                 owner=params.kafka_user,
                 content=InlineTemplate(params.kafka_env_sh_template))

            File(format("{kafka_conf_dir}/kafka_jaas.conf"),
                 group=params.user_group,
                 owner=params.kafka_user,
                 content=Template("kafka_jaas.conf.j2"))

    if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
            params.namenode_host) > 1:
        XmlConfig(
            "hdfs-site.xml",
            conf_dir=params.conf_dir,
            configurations=params.config['configurations']['hdfs-site'],
            configuration_attributes=params.config['configurationAttributes']
            ['hdfs-site'],
            owner=params.metadata_user,
            group=params.user_group,
            mode=0644)
    else:
        File(format('{conf_dir}/hdfs-site.xml'), action="delete")

    if params.stack_supports_atlas_core_site and params.has_namenode:
        XmlConfig(
            "core-site.xml",
            conf_dir=params.conf_dir,
            configurations=params.config['configurations']['core-site'],
            configuration_attributes=params.config['configurationAttributes']
            ['core-site'],
            owner=params.metadata_user,
            group=params.user_group,
            mode=0644,
            xml_include_file=params.mount_table_xml_inclusion_file_full_path)

        if params.mount_table_content:
            File(params.mount_table_xml_inclusion_file_full_path,
                 owner=params.metadata_user,
                 group=params.user_group,
                 content=params.mount_table_content,
                 mode=0644)

    Directory(
        format('{metadata_home}/'),
        owner=params.metadata_user,
        group=params.user_group,
        recursive_ownership=True,
    )