Ejemplo n.º 1
0
def new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user,
                    keytab_file, principal, hostname):
    """
  Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution
  """
    now = datetime.now()
    temp_kinit_cache_fd, temp_kinit_cache_filename = mkstemp(dir=temp_dir)
    command = "%s -c %s -kt %s %s" % \
              (kinit_path, temp_kinit_cache_filename, keytab_file,
               principal.replace("_HOST", hostname))

    os.close(temp_kinit_cache_fd)

    try:
        # Ensure the proper user owns this file
        File(temp_kinit_cache_filename, owner=exec_user, mode=0600)

        # Execute the kinit
        Execute(command, user=exec_user)

        with open(file_path, 'w+') as cache_file:
            result = {key: {"last_successful_execution": str(now)}}
            json.dump(result, cache_file)
    finally:
        File(temp_kinit_cache_filename, action='delete')
Ejemplo n.º 2
0
def metadata():
    import params

    Directory([params.pid_dir],
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    Directory(params.log_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    Directory(params.data_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    Directory(params.expanded_war_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    File(format("{expanded_war_dir}/atlas.war"),
         content=StaticFile(format('{metadata_home}/server/webapp/atlas.war')))

    PropertiesFile(format('{conf_dir}/application.properties'),
                   properties=params.application_properties,
                   mode=0644,
                   owner=params.metadata_user,
                   group=params.user_group)

    File(format("{conf_dir}/atlas-env.sh"),
         owner=params.metadata_user,
         group=params.user_group,
         mode=0755,
         content=InlineTemplate(params.metadata_env_content))

    File(format("{conf_dir}/atlas-log4j.xml"),
         mode=0644,
         owner=params.metadata_user,
         group=params.user_group,
         content=StaticFile('atlas-log4j.xml'))
Ejemplo n.º 3
0
def setup_livy(env, type, upgrade_type=None, action=None):
    import params

    Directory([params.livy2_pid_dir, params.livy2_log_dir],
              owner=params.livy2_user,
              group=params.user_group,
              mode=0775,
              create_parents=True)
    if type == 'server' and action == 'config':
        params.HdfsResource(params.livy2_hdfs_user_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.livy2_user,
                            mode=0775)
        params.HdfsResource(None, action="execute")

    # create livy-env.sh in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'livy-env.sh'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=InlineTemplate(params.livy2_env_sh),
        mode=0644,
    )

    # create livy.conf in etc/conf dir
    PropertiesFile(
        format("{livy2_conf}/livy.conf"),
        properties=params.config['configurations']['livy2-conf'],
        key_value_delimiter=" ",
        owner=params.livy2_user,
        group=params.livy2_group,
    )

    # create log4j.properties in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'log4j.properties'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=params.livy2_log4j_properties,
        mode=0644,
    )

    # create spark-blacklist.properties in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=params.livy2_spark_blacklist_properties,
        mode=0644,
    )

    Directory(
        params.livy2_logs_dir,
        owner=params.livy2_user,
        group=params.livy2_group,
        mode=0755,
    )
Ejemplo n.º 4
0
    def stop(self, env, upgrade_type=None):
        import params

        env.set_params(params)
        daemon_cmd = format(
            'source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}'
        )

        # If the pid dir doesn't exist, this means either
        # 1. The user just added Atlas service and issued a restart command (stop+start). So stop should be a no-op
        # since there's nothing to stop.
        # OR
        # 2. The user changed the value of the pid dir config and incorrectly issued a restart command.
        # In which case the stop command cannot do anything since Ambari doesn't know which process to kill.
        # The start command will spawn another instance.
        # The user should have issued a stop, changed the config, and then started it.
        if not os.path.isdir(params.pid_dir):
            Logger.info(
                "*******************************************************************"
            )
            Logger.info(
                "Will skip the stop command since this is the first time stopping/restarting Atlas "
                "and the pid dir does not exist, %s\n" % params.pid_dir)
            return

        try:
            Execute(
                daemon_cmd,
                user=params.metadata_user,
            )
        except:
            show_logs(params.log_dir, params.metadata_user)
            raise

        File(params.pid_file, action="delete")
Ejemplo n.º 5
0
    def setup_config(self, env):
        import params
        env.set_params(params)
        XmlConfig(
            "hdfs-site.xml",
            conf_dir=params.hadoop_conf_dir,
            configurations=params.config['configurations']['hdfs-site'],
            configuration_attributes=params.config['configuration_attributes']
            ['hdfs-site'],
            owner=params.hdfs_user,
            group=params.user_group)
        XmlConfig(
            "core-site.xml",
            conf_dir=params.hadoop_conf_dir,
            configurations=params.config['configurations']['core-site'],
            configuration_attributes=params.config['configuration_attributes']
            ['core-site'],
            owner=params.hdfs_user,
            group=params.user_group,
            mode=0644)

        File(
            format("{params.ambari_libs_dir}/fast-hdfs-resource.jar"),
            mode=0644,
            content=StaticFile(
                "/var/lib/ambari-agent/cache/stack-hooks/before-START/files/fast-hdfs-resource.jar"
            ))
Ejemplo n.º 6
0
def update_credential_provider_path(config, config_type, dest_provider_path,
                                    file_owner, file_group):
    """
  Copies the JCEKS file for the specified config from the default location to the given location,
  and sets the ACLs for the specified owner and group. Also updates the config type's configuration
  hadoop credential store provider with the copied file name.
  :param config: configurations['configurations'][config_type]
  :param config_type: Like hive-site, oozie-site, etc.
  :param dest_provider_path: The full path to the file where the JCEKS provider file is to be copied to.
  :param file_owner: File owner
  :param file_group: Group
  :return: A copy of the config that was modified or the input config itself if nothing was modified.
  """
    # Get the path to the provider <config_type>.jceks
    if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in config:
        provider_path = config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME]
        src_provider_path = provider_path[len('jceks://file'):]
        File(dest_provider_path,
             owner=file_owner,
             group=file_group,
             mode=0640,
             content=StaticFile(src_provider_path))
        # make a copy of the config dictionary since it is read-only
        config_copy = config.copy()
        # overwrite the provider path with the path specified
        config_copy[
            HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file{0}'.format(
                dest_provider_path)
        return config_copy
    return config
    def configure_airflow(self, env):
        import params
        env.set_params(params)

        File("/etc/systemd/system/airflow_webserver.service",
             content=Template("webserver.j2", configurations=params),
             owner='root',
             group='root',
             mode=0o0600)

        File("/etc/systemd/system/airflow_scheduler.service",
             content=Template("scheduler.j2", configurations=params),
             owner='root',
             group='root',
             mode=0o0600)

        Execute('sudo systemctl daemon-reload')
 def configure_ac(self, env):
     import params
     env.set_params(params)
     print("!!!!!!!!!!!!!!!!!!!!!!!creating file!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
     File("/etc/systemd/system/airflow_portforward.service",
          content=Template("portforward_service.j2", configurations=params),
          owner='root',
          group='root',
          mode=0o0600
          )
     Execute('sudo systemctl daemon-reload')
Ejemplo n.º 9
0
    def configure_ac(self, env):
        import params
        env.set_params(params)

        File("/etc/systemd/system/jupyter_portforward.service",
             content=Template("portforward_service_jupyter.j2",
                              configurations=params),
             owner=params.anaconda_user,
             group=params.anaconda_group,
             mode=0o0600)
        Execute('sudo systemctl daemon-reload')
Ejemplo n.º 10
0
  def stop(self, env, upgrade_type=None):
    import params
    env.set_params(params)
    daemon_cmd = format('source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}')

    try:
      Execute(daemon_cmd,
              user=params.metadata_user,
      )
    except:
      show_logs(params.log_dir, params.metadata_user)
      raise

    File(params.pid_file, action="delete")
Ejemplo n.º 11
0
def install_jce_policy():
    import params
    jcePolicyInfo = JcePolicyInfo(params.java_home)

    if jcePolicyInfo.is_unlimited_key_jce_policy():
        major, minor = java_version()
        Logger.info(
            "The unlimited key JCE policy is required, and appears to have been installed."
        )
        java_security_dir = format("{java_home}/jre/lib/security")
        Logger.debug("Removing existing JCE policy JAR files: {0}.".format(
            java_security_dir))
        File(format("{java_security_dir}/US_export_policy.jar"),
             action="delete")
        File(format("{java_security_dir}/local_policy.jar"), action="delete")
        src_url = jce_zip_target = ''
        if minor == 7:  # jdk7
            src_url = 'http://yum.example.com/hadoop/UnlimitedJCEPolicyJDK7.zip'
            jce_zip_target = '/tmp/UnlimitedJCEPolicyJDK7.zip'
        elif minor == 8:  # jdk8
            src_url = 'http://yum.example.com/hadoop/jce_policy-8.zip'
            jce_zip_target = '/tmp/jce_policy-8.zip'

        if src_url and jce_zip_target:
            Execute('wget ' + src_url + ' -O ' + jce_zip_target)
            Logger.debug(
                "Unzipping the unlimited key JCE policy files from {0} into {1}."
                .format(jce_zip_target, java_security_dir))
            extract_cmd = ("unzip", "-o", "-j", "-q", jce_zip_target, "-d",
                           java_security_dir)
            Execute(
                extract_cmd,
                only_if=format(
                    "test -e {java_security_dir} && test -f {jce_zip_target}"),
                path=['/bin/', '/usr/bin'],
                sudo=True)
    def configure_ac(self, env):
        import params
        env.set_params(params)

        conf = dict()
        for key in params.config['configurations']['jupyter-env']:
            conf[key] = params.config['configurations']['jupyter-env'][key]

        conf['jupyter_password'] = params.hashText(conf['jupyter_password'])

        print("----------------------------\n", conf, "\n------------------------------------------------")

        File("{0}jupyter_notebook_config.py".format(params.config_dir),
             content=Template("jupyter_notebook_config.py.j2",
                              configurations=conf),
             owner=params.anaconda_user,
             group=params.anaconda_group,
             mode=0o0600
             )
Ejemplo n.º 13
0
def metadata():
    import params

    Directory([params.pid_dir],
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              recursive=True
    )

    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              recursive=True
    )

    Directory(params.log_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              recursive=True
    )

    Directory(params.data_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              recursive=True
    )

    Directory(params.expanded_war_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              recursive=True
    )

    metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
    if not os.path.isfile(metadata_war_file):
        raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))

    Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
    shutil.copy2(metadata_war_file, params.expanded_war_dir)

    File(format('{conf_dir}/application.properties'),
         content=InlineTemplate(params.application_properties_content),
         mode=0644,
         owner=params.metadata_user,
         group=params.user_group
    )

    File(format("{conf_dir}/metadata-env.sh"),
         owner=params.metadata_user,
         group=params.user_group,
         mode=0755,
         content=InlineTemplate(params.metadata_env_content)
    )

    File(format("{conf_dir}/log4j.xml"),
         mode=0644,
         owner=params.metadata_user,
         group=params.user_group,
         content=StaticFile('log4j.xml')
    )
Ejemplo n.º 14
0
def setup_livy(env, type, upgrade_type=None, action=None):
    import params

    Directory([params.livy2_pid_dir, params.livy2_log_dir],
              owner=params.livy2_user,
              group=params.user_group,
              mode=0775,
              cd_access='a',
              create_parents=True)
    if type == 'server' and action == 'config':
        params.HdfsResource(params.livy2_hdfs_user_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.livy2_user,
                            mode=0775)
        params.HdfsResource(None, action="execute")

        if params.livy2_recovery_store == 'filesystem':
            params.HdfsResource(params.livy2_recovery_dir,
                                type="directory",
                                action="create_on_execute",
                                owner=params.livy2_user,
                                mode=0700)
            params.HdfsResource(None, action="execute")

        generate_logfeeder_input_config(
            'spark2',
            Template("input.config-spark2.json.j2", extra_imports=[default]))

    # create livy-env.sh in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'livy-env.sh'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=InlineTemplate(params.livy2_env_sh),
        mode=0644,
    )

    # create livy-client.conf in etc/conf dir
    PropertiesFile(
        format("{livy2_conf}/livy-client.conf"),
        properties=params.config['configurations']['livy2-client-conf'],
        key_value_delimiter=" ",
        owner=params.livy2_user,
        group=params.livy2_group,
    )

    # create livy.conf in etc/conf dir
    PropertiesFile(
        format("{livy2_conf}/livy.conf"),
        properties=params.config['configurations']['livy2-conf'],
        key_value_delimiter=" ",
        owner=params.livy2_user,
        group=params.livy2_group,
    )

    # create log4j.properties in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'log4j.properties'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=params.livy2_log4j_properties,
        mode=0644,
    )

    # create spark-blacklist.properties in etc/conf dir
    File(
        os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
        owner=params.livy2_user,
        group=params.livy2_group,
        content=params.livy2_spark_blacklist_properties,
        mode=0644,
    )

    Directory(
        params.livy2_logs_dir,
        owner=params.livy2_user,
        group=params.livy2_group,
        mode=0755,
    )
Ejemplo n.º 15
0
def metadata(type='server'):
    import params

    # Needed by both Server and Client
    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents=True)

    if type == "server":
        Directory([params.pid_dir],
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(format('{conf_dir}/solr'),
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.log_dir,
                  mode=0755,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.data_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        Directory(params.expanded_war_dir,
                  mode=0644,
                  cd_access='a',
                  owner=params.metadata_user,
                  group=params.user_group,
                  create_parents=True)
        File(format("{expanded_war_dir}/atlas.war"),
             content=StaticFile(
                 format('{metadata_home}/server/webapp/atlas.war')))
        File(format("{conf_dir}/atlas-log4j.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_log4j_content))
        File(format("{conf_dir}/atlas-env.sh"),
             owner=params.metadata_user,
             group=params.user_group,
             mode=0755,
             content=InlineTemplate(params.metadata_env_content))
        File(format("{conf_dir}/solr/solrconfig.xml"),
             mode=0644,
             owner=params.metadata_user,
             group=params.user_group,
             content=InlineTemplate(params.metadata_solrconfig_content))

    # Needed by both Server and Client
    PropertiesFile(format('{conf_dir}/{conf_file}'),
                   properties=params.application_properties,
                   mode=0644,
                   owner=params.metadata_user,
                   group=params.user_group)

    if type == 'server' and params.search_backend_solr and params.has_logsearch_solr:
        solr_cloud_util.setup_solr_client(params.config)

        random_num = random.random()

        upload_conf_set('basic_configs', random_num)

        create_collection('vertex_index', 'basic_configs')
        create_collection('edge_index', 'basic_configs')
        create_collection('fulltext_index', 'basic_configs')

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                       owner=params.metadata_user)
Ejemplo n.º 16
0
def metadata():
    import params

    Directory([params.pid_dir],
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
    )

    Directory(params.conf_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
    )

    Directory(params.log_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
    )

    Directory(params.atlas_hbase_log_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
              )

    Directory(params.atlas_hbase_data_dir,
              mode=0755,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
              )

    Directory(params.data_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
    )

    Directory(params.expanded_war_dir,
              mode=0644,
              cd_access='a',
              owner=params.metadata_user,
              group=params.user_group,
              create_parents = True
    )

    File(format("{expanded_war_dir}/atlas.war"),
         content = StaticFile(format('{metadata_home}/server/webapp/atlas.war'))
    )

    PropertiesFile(format('{conf_dir}/{conf_file}'),
         properties = params.application_properties,
         mode=0644,
         owner=params.metadata_user,
         group=params.user_group
    )

    File(format("{conf_dir}/atlas-env.sh"),
         owner=params.metadata_user,
         group=params.user_group,
         mode=0755,
         content=InlineTemplate(params.metadata_env_content)
    )

    File(format("{conf_dir}/atlas-log4j.xml"),
         mode=0644,
         owner=params.metadata_user,
         group=params.user_group,
         content=InlineTemplate(params.metadata_log4j_content)
    )

    # hbase-site for embedded hbase used by Atlas
    XmlConfig( "hbase-site.xml",
           conf_dir = params.atlas_hbase_conf_dir,
           configurations = params.config['configurations']['atlas-hbase-site'],
           configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'],
           owner = params.metadata_user,
           group = params.user_group
           )

    if params.security_enabled:
        TemplateConfig(format(params.atlas_jaas_file),
                         owner=params.metadata_user)