Example #1
0
def generate_configs(args, host, job_name, instance_id):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["core-site.xml"]
    )
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["hdfs-site.xml"]
    )
    mapred_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["mapred-site.xml"]
    )
    yarn_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["yarn-site.xml"]
    )
    hadoop_metrics2_properties = generate_metrics_config(args, host, job_name, instance_id)

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "mapred-site.xml": mapred_site_xml,
        "yarn-site.xml": yarn_site_xml,
        "hadoop-metrics2.properties": hadoop_metrics2_properties,
    }
    config_files.update(args.yarn_config.configuration.raw_files)

    return config_files
Example #2
0
def generate_client_config(args, artifact, version):
    config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster,
                                        artifact, version)
    master_host = args.hbase_config.jobs["master"].hosts[0].ip
    config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster,
                                        artifact, version)
    deploy_utils.write_file(
        "%s/hbase-site.xml" % config_path,
        deploy_utils.generate_site_xml(
            args,
            args.hbase_config.configuration.generated_files["hbase-site.xml"]))
    deploy_utils.write_file(
        "%s/hadoop-metrics.properties" % config_path,
        generate_metrics_config(args, master_host, "master"))
    deploy_utils.write_file(
        "%s/core-site.xml" % config_path,
        deploy_utils.generate_site_xml(
            args,
            args.hbase_config.configuration.generated_files["core-site.xml"]))
    deploy_utils.write_file(
        "%s/hdfs-site.xml" % config_path,
        deploy_utils.generate_site_xml(
            args,
            args.hbase_config.configuration.generated_files["hdfs-site.xml"]))
    args.zookeeper_config.parse_generated_config_files(args)
    deploy_utils.write_file("%s/jaas.conf" % config_path,
                            deploy_zookeeper.generate_client_jaas_config(args))
    deploy_utils.write_file(
        "%s/krb5.conf" % config_path,
        args.hbase_config.configuration.raw_files["krb5.conf"])
    update_hbase_env_sh(args, artifact, version)
Example #3
0
def generate_configs(args, host, job_name):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["hdfs-site.xml"])
    mapred_site_xml = deploy_utils.generate_site_xml(
        args,
        args.yarn_config.configuration.generated_files["mapred-site.xml"])
    yarn_site_xml = deploy_utils.generate_site_xml(
        args, args.yarn_config.configuration.generated_files["yarn-site.xml"])
    hadoop_metrics2_properties = generate_metrics_config(args, host, job_name)
    yarn_raw_files = args.yarn_config.configuration.raw_files

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "mapred-site.xml": mapred_site_xml,
        "yarn-site.xml": yarn_site_xml,
        "hadoop-metrics2.properties": hadoop_metrics2_properties,
        "configuration.xsl": yarn_raw_files["configuration.xsl"],
        "log4j.xml": yarn_raw_files["log4j.xml"],
        "krb5.conf": yarn_raw_files["krb5.conf"],
        "excludes": yarn_raw_files["excludes"],
    }
    return config_files
Example #4
0
def generate_client_config(args, artifact, version):
  config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root,
      args.cluster, artifact, version)
  deploy_utils.write_file("%s/mapred-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.yarn_config.configuration.generated_files["mapred-site.xml"]))
  deploy_utils.write_file("%s/yarn-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.yarn_config.configuration.generated_files["yarn-site.xml"]))
  deploy_utils.write_file("%s/krb5.conf" % config_path,
      args.yarn_config.configuration.raw_files["krb5.conf"])
  deploy_hdfs.update_hadoop_env_sh(args, artifact, version, "YARN_OPTS")
Example #5
0
def generate_client_config(args, artifact, version):
  config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root,
      args.cluster, artifact, version)
  deploy_utils.write_file("%s/mapred-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.yarn_config.configuration.generated_files["mapred-site.xml"]))
  deploy_utils.write_file("%s/yarn-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.yarn_config.configuration.generated_files["yarn-site.xml"]))
  deploy_utils.write_file("%s/krb5.conf" % config_path,
      args.yarn_config.configuration.raw_files["krb5.conf"])
  deploy_hdfs.update_hadoop_env_sh(args, artifact, version, "YARN_OPTS")
Example #6
0
def generate_configs(args):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.impala_config.configuration.generated_files["core-site.xml"]
    )
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.impala_config.configuration.generated_files["hdfs-site.xml"]
    )
    hive_site_xml = deploy_utils.generate_site_xml(
        args, args.impala_config.configuration.generated_files["hive-site.xml"]
    )

    config_files = {"core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hive-site.xml": hive_site_xml}
    config_files.update(args.impala_config.configuration.raw_files)

    return config_files
Example #7
0
def generate_client_config(args, artifact, version):
  config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root,
      args.cluster, artifact, version)
  deploy_utils.write_file("%s/core-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hdfs_config.configuration.generated_files["core-site.xml"]))
  deploy_utils.write_file("%s/hdfs-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hdfs_config.configuration.generated_files["hdfs-site.xml"]))
  deploy_utils.write_file("%s/hadoop-metrics2.properties" % config_path,
      generate_metrics_config(args, args.hdfs_config.jobs["namenode"].hosts[0],
        "namenode"))
  deploy_utils.write_file("%s/krb5.conf" % config_path,
      args.hdfs_config.configuration.raw_files["krb5.conf"])
  update_hadoop_env_sh(args, artifact, version, "HADOOP_OPTS")
Example #8
0
def generate_configs(args, host, job_name):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.hdfs_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.hdfs_config.configuration.generated_files["hdfs-site.xml"])
  hadoop_metrics2_properties = generate_metrics_config(args, host, job_name)

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hadoop-metrics2.properties": hadoop_metrics2_properties,
  }
  config_files.update(args.hdfs_config.configuration.raw_files)

  return config_files
Example #9
0
def generate_client_config(args, artifact, version):
  config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root,
      args.cluster, artifact, version)
  deploy_utils.write_file("%s/core-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hdfs_config.configuration.generated_files["core-site.xml"]))
  deploy_utils.write_file("%s/hdfs-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hdfs_config.configuration.generated_files["hdfs-site.xml"]))
  deploy_utils.write_file("%s/hadoop-metrics2.properties" % config_path,
      generate_metrics_config(args, args.hdfs_config.jobs["namenode"].hosts[0].ip,
        "namenode"))
  deploy_utils.write_file("%s/krb5.conf" % config_path,
      args.hdfs_config.configuration.raw_files["krb5.conf"])
  update_hadoop_env_sh(args, artifact, version, "HADOOP_OPTS")
Example #10
0
def generate_configs(args, host, job_name):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.hdfs_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"])
    hadoop_metrics2_properties = generate_metrics_config(args, host, job_name)

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "hadoop-metrics2.properties": hadoop_metrics2_properties,
    }
    config_files.update(args.hdfs_config.configuration.raw_files)

    return config_files
Example #11
0
def generate_configs(args):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["hdfs-site.xml"])
  hive_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["hive-site.xml"])
  log4j_xml = args.impala_config.configuration.raw_files["log4j.xml"]

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hive-site.xml": hive_site_xml,
    "log4j.xml": log4j_xml,
  }
  return config_files
Example #12
0
def generate_configs(args):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["hdfs-site.xml"])
  hive_site_xml = deploy_utils.generate_site_xml(args,
    args.impala_config.configuration.generated_files["hive-site.xml"])

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hive-site.xml": hive_site_xml,
  }
  config_files.update(args.impala_config.configuration.raw_files)

  return config_files
Example #13
0
def generate_configs(args):
    core_site_xml = deploy_utils.generate_site_xml(
        args,
        args.impala_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args,
        args.impala_config.configuration.generated_files["hdfs-site.xml"])
    hive_site_xml = deploy_utils.generate_site_xml(
        args,
        args.impala_config.configuration.generated_files["hive-site.xml"])
    log4j_xml = args.impala_config.configuration.raw_files["log4j.xml"]

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "hive-site.xml": hive_site_xml,
        "log4j.xml": log4j_xml,
    }
    return config_files
Example #14
0
def generate_configs(args, host, job_name):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.hdfs_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.hdfs_config.configuration.generated_files["hdfs-site.xml"])
  hadoop_metrics2_properties = generate_metrics_config(args, host, job_name)
  hdfs_raw_files = args.hdfs_config.configuration.raw_files

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hadoop-metrics2.properties": hadoop_metrics2_properties,
    "configuration.xsl": hdfs_raw_files["configuration.xsl"],
    "log4j.xml": hdfs_raw_files["log4j.xml"],
    "krb5.conf": hdfs_raw_files["krb5.conf"],
    "rackinfo.txt": hdfs_raw_files["rackinfo.txt"],
    "excludes": hdfs_raw_files["excludes"],
    "hadoop-groups.conf": hdfs_raw_files["hadoop-groups.conf"],
  }
  return config_files
def generate_configs(args, host, job_name, instance_id):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["hdfs-site.xml"])
  hbase_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["hbase-site.xml"])
  hadoop_metrics_properties = generate_metrics_config(args, host, job_name, instance_id)
  zk_jaas_conf = generate_zk_jaas_config(args)

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hbase-site.xml": hbase_site_xml,
    "hadoop-metrics.properties": hadoop_metrics_properties,
    "jaas.conf": zk_jaas_conf,
  }
  config_files.update(args.hbase_config.configuration.raw_files)

  return config_files
Example #16
0
def generate_configs(args, host, job_name, instance_id):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["hdfs-site.xml"])
  mapred_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["mapred-site.xml"])
  yarn_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["yarn-site.xml"])
  hadoop_metrics2_properties = generate_metrics_config(args, host, job_name, instance_id)

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "mapred-site.xml": mapred_site_xml,
    "yarn-site.xml": yarn_site_xml,
    "hadoop-metrics2.properties": hadoop_metrics2_properties,
  }
  config_files.update(args.yarn_config.configuration.raw_files)

  return config_files
Example #17
0
def generate_configs(args, host, job_name, instance_id):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["hdfs-site.xml"])
  hbase_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["hbase-site.xml"])
  galaxy_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["galaxy-site.xml"])
  zookeeper_properties = deploy_utils.generate_properties_file(args,
      args.fds_config.configuration.generated_files["zookeeper.properties"])
  mapred_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["mapred-site.xml"])
  yarn_site_xml = deploy_utils.generate_site_xml(args,
    args.fds_config.configuration.generated_files["yarn-site.xml"])
  passport_properties = deploy_utils.generate_properties_file(args,
      args.fds_config.configuration.generated_files["passport.properties"])

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hbase-site.xml": hbase_site_xml,
    "galaxy-site.xml": galaxy_site_xml,
    "zookeeper.properties": zookeeper_properties,
    "mapred-site.xml": mapred_site_xml,
    "yarn-site.xml": yarn_site_xml,
    "passport.properties": passport_properties,
  }
  config_files.update(args.fds_config.configuration.raw_files)

  return config_files
Example #18
0
def generate_configs(args, host, job_name, instance_id):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["hdfs-site.xml"])
    hbase_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["hbase-site.xml"])
    galaxy_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["galaxy-site.xml"])
    zookeeper_properties = deploy_utils.generate_properties_file(
        args,
        args.fds_config.configuration.generated_files["zookeeper.properties"])
    mapred_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["mapred-site.xml"])
    yarn_site_xml = deploy_utils.generate_site_xml(
        args, args.fds_config.configuration.generated_files["yarn-site.xml"])
    passport_properties = deploy_utils.generate_properties_file(
        args,
        args.fds_config.configuration.generated_files["passport.properties"])

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "hbase-site.xml": hbase_site_xml,
        "galaxy-site.xml": galaxy_site_xml,
        "zookeeper.properties": zookeeper_properties,
        "mapred-site.xml": mapred_site_xml,
        "yarn-site.xml": yarn_site_xml,
        "passport.properties": passport_properties,
    }
    config_files.update(args.fds_config.configuration.raw_files)

    return config_files
Example #19
0
def generate_client_config(args, artifact, version):
  config_path = "%s/%s/%s-%s/conf" % (args.package_root,
      args.cluster, artifact, version)
  master_host = args.hbase_config.jobs["master"].hosts[0]
  config_path = "%s/%s/%s-%s/conf" % (args.package_root,
      args.cluster, artifact, version)
  deploy_utils.write_file("%s/hbase-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hbase_config.configuration.generated_files["hbase-site.xml"]))
  deploy_utils.write_file("%s/hadoop-metrics.properties" % config_path,
      generate_metrics_config(args, master_host, "master"))
  deploy_utils.write_file("%s/core-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hbase_config.configuration.generated_files["core-site.xml"]))
  deploy_utils.write_file("%s/hdfs-site.xml" % config_path,
      deploy_utils.generate_site_xml(args,
        args.hbase_config.configuration.generated_files["hdfs-site.xml"]))
  deploy_utils.write_file("%s/jaas.conf" % config_path,
      deploy_zookeeper.generate_client_jaas_config(args))
  deploy_utils.write_file("%s/krb5.conf" % config_path,
      args.hbase_config.configuration.raw_files["krb5.conf"])
  update_hbase_env_sh(args, artifact, version)
Example #20
0
def generate_configs(args, host, job_name):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["hdfs-site.xml"])
  hbase_site_xml = deploy_utils.generate_site_xml(args,
    args.hbase_config.configuration.generated_files["hbase-site.xml"])
  hadoop_metrics_properties = generate_metrics_config(args, host, job_name)
  zk_jaas_conf = generate_zk_jaas_config(args)
  hbase_raw_files = args.hbase_config.configuration.raw_files

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "hbase-site.xml": hbase_site_xml,
    "hadoop-metrics.properties": hadoop_metrics_properties,
    "jaas.conf": zk_jaas_conf,
    "configuration.xsl": hbase_raw_files["configuration.xsl"],
    "log4j.xml": hbase_raw_files["log4j.xml"],
    "krb5.conf": hbase_raw_files["krb5.conf"],
  }
  return config_files
Example #21
0
def generate_configs(args, host, job_name, instance_id):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.hbase_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.hbase_config.configuration.generated_files["hdfs-site.xml"])
    hbase_site_xml = deploy_utils.generate_site_xml(
        args,
        args.hbase_config.configuration.generated_files["hbase-site.xml"])
    hadoop_metrics_properties = generate_metrics_config(
        args, host, job_name, instance_id)
    zk_jaas_conf = generate_zk_jaas_config(args)

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "hbase-site.xml": hbase_site_xml,
        "hadoop-metrics.properties": hadoop_metrics_properties,
        "jaas.conf": zk_jaas_conf,
    }
    config_files.update(args.hbase_config.configuration.raw_files)

    return config_files
Example #22
0
def generate_configs(args, host, job_name):
    core_site_xml = deploy_utils.generate_site_xml(
        args, args.hbase_config.configuration.generated_files["core-site.xml"])
    hdfs_site_xml = deploy_utils.generate_site_xml(
        args, args.hbase_config.configuration.generated_files["hdfs-site.xml"])
    hbase_site_xml = deploy_utils.generate_site_xml(
        args,
        args.hbase_config.configuration.generated_files["hbase-site.xml"])
    hadoop_metrics_properties = generate_metrics_config(args, host, job_name)
    zk_jaas_conf = generate_zk_jaas_config(args)
    hbase_raw_files = args.hbase_config.configuration.raw_files

    config_files = {
        "core-site.xml": core_site_xml,
        "hdfs-site.xml": hdfs_site_xml,
        "hbase-site.xml": hbase_site_xml,
        "hadoop-metrics.properties": hadoop_metrics_properties,
        "jaas.conf": zk_jaas_conf,
        "configuration.xsl": hbase_raw_files["configuration.xsl"],
        "log4j.xml": hbase_raw_files["log4j.xml"],
        "krb5.conf": hbase_raw_files["krb5.conf"],
    }
    return config_files
Example #23
0
def generate_configs(args, host, job_name):
  core_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["core-site.xml"])
  hdfs_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["hdfs-site.xml"])
  mapred_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["mapred-site.xml"])
  yarn_site_xml = deploy_utils.generate_site_xml(args,
    args.yarn_config.configuration.generated_files["yarn-site.xml"])
  hadoop_metrics2_properties = generate_metrics_config(args, host, job_name)
  yarn_raw_files = args.yarn_config.configuration.raw_files

  config_files = {
    "core-site.xml": core_site_xml,
    "hdfs-site.xml": hdfs_site_xml,
    "mapred-site.xml": mapred_site_xml,
    "yarn-site.xml": yarn_site_xml,
    "hadoop-metrics2.properties": hadoop_metrics2_properties,
    "configuration.xsl": yarn_raw_files["configuration.xsl"],
    "log4j.xml": yarn_raw_files["log4j.xml"],
    "krb5.conf": yarn_raw_files["krb5.conf"],
    "excludes": yarn_raw_files["excludes"],
  }
  return config_files