def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["core-site.xml"] ) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["hdfs-site.xml"] ) mapred_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["mapred-site.xml"] ) yarn_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["yarn-site.xml"] ) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name, instance_id) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, } config_files.update(args.yarn_config.configuration.raw_files) return config_files
def generate_client_config(args, artifact, version): config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster, artifact, version) master_host = args.hbase_config.jobs["master"].hosts[0].ip config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster, artifact, version) deploy_utils.write_file( "%s/hbase-site.xml" % config_path, deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hbase-site.xml"])) deploy_utils.write_file( "%s/hadoop-metrics.properties" % config_path, generate_metrics_config(args, master_host, "master")) deploy_utils.write_file( "%s/core-site.xml" % config_path, deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["core-site.xml"])) deploy_utils.write_file( "%s/hdfs-site.xml" % config_path, deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hdfs-site.xml"])) args.zookeeper_config.parse_generated_config_files(args) deploy_utils.write_file("%s/jaas.conf" % config_path, deploy_zookeeper.generate_client_jaas_config(args)) deploy_utils.write_file( "%s/krb5.conf" % config_path, args.hbase_config.configuration.raw_files["krb5.conf"]) update_hbase_env_sh(args, artifact, version)
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["hdfs-site.xml"]) mapred_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["mapred-site.xml"]) yarn_site_xml = deploy_utils.generate_site_xml( args, args.yarn_config.configuration.generated_files["yarn-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name) yarn_raw_files = args.yarn_config.configuration.raw_files config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, "configuration.xsl": yarn_raw_files["configuration.xsl"], "log4j.xml": yarn_raw_files["log4j.xml"], "krb5.conf": yarn_raw_files["krb5.conf"], "excludes": yarn_raw_files["excludes"], } return config_files
def generate_client_config(args, artifact, version): config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root, args.cluster, artifact, version) deploy_utils.write_file("%s/mapred-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["mapred-site.xml"])) deploy_utils.write_file("%s/yarn-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["yarn-site.xml"])) deploy_utils.write_file("%s/krb5.conf" % config_path, args.yarn_config.configuration.raw_files["krb5.conf"]) deploy_hdfs.update_hadoop_env_sh(args, artifact, version, "YARN_OPTS")
def generate_configs(args): core_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["core-site.xml"] ) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["hdfs-site.xml"] ) hive_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["hive-site.xml"] ) config_files = {"core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hive-site.xml": hive_site_xml} config_files.update(args.impala_config.configuration.raw_files) return config_files
def generate_client_config(args, artifact, version): config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root, args.cluster, artifact, version) deploy_utils.write_file("%s/core-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["core-site.xml"])) deploy_utils.write_file("%s/hdfs-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"])) deploy_utils.write_file("%s/hadoop-metrics2.properties" % config_path, generate_metrics_config(args, args.hdfs_config.jobs["namenode"].hosts[0], "namenode")) deploy_utils.write_file("%s/krb5.conf" % config_path, args.hdfs_config.configuration.raw_files["krb5.conf"]) update_hadoop_env_sh(args, artifact, version, "HADOOP_OPTS")
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, } config_files.update(args.hdfs_config.configuration.raw_files) return config_files
def generate_client_config(args, artifact, version): config_path = "%s/%s/%s-%s/etc/hadoop" % (args.package_root, args.cluster, artifact, version) deploy_utils.write_file("%s/core-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["core-site.xml"])) deploy_utils.write_file("%s/hdfs-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"])) deploy_utils.write_file("%s/hadoop-metrics2.properties" % config_path, generate_metrics_config(args, args.hdfs_config.jobs["namenode"].hosts[0].ip, "namenode")) deploy_utils.write_file("%s/krb5.conf" % config_path, args.hdfs_config.configuration.raw_files["krb5.conf"]) update_hadoop_env_sh(args, artifact, version, "HADOOP_OPTS")
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml( args, args.hdfs_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, } config_files.update(args.hdfs_config.configuration.raw_files) return config_files
def generate_configs(args): core_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["hdfs-site.xml"]) hive_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["hive-site.xml"]) log4j_xml = args.impala_config.configuration.raw_files["log4j.xml"] config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hive-site.xml": hive_site_xml, "log4j.xml": log4j_xml, } return config_files
def generate_configs(args): core_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["hdfs-site.xml"]) hive_site_xml = deploy_utils.generate_site_xml(args, args.impala_config.configuration.generated_files["hive-site.xml"]) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hive-site.xml": hive_site_xml, } config_files.update(args.impala_config.configuration.raw_files) return config_files
def generate_configs(args): core_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["hdfs-site.xml"]) hive_site_xml = deploy_utils.generate_site_xml( args, args.impala_config.configuration.generated_files["hive-site.xml"]) log4j_xml = args.impala_config.configuration.raw_files["log4j.xml"] config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hive-site.xml": hive_site_xml, "log4j.xml": log4j_xml, } return config_files
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.hdfs_config.configuration.generated_files["hdfs-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name) hdfs_raw_files = args.hdfs_config.configuration.raw_files config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, "configuration.xsl": hdfs_raw_files["configuration.xsl"], "log4j.xml": hdfs_raw_files["log4j.xml"], "krb5.conf": hdfs_raw_files["krb5.conf"], "rackinfo.txt": hdfs_raw_files["rackinfo.txt"], "excludes": hdfs_raw_files["excludes"], "hadoop-groups.conf": hdfs_raw_files["hadoop-groups.conf"], } return config_files
def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hbase-site.xml"]) hadoop_metrics_properties = generate_metrics_config(args, host, job_name, instance_id) zk_jaas_conf = generate_zk_jaas_config(args) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "hadoop-metrics.properties": hadoop_metrics_properties, "jaas.conf": zk_jaas_conf, } config_files.update(args.hbase_config.configuration.raw_files) return config_files
def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["hdfs-site.xml"]) mapred_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["mapred-site.xml"]) yarn_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["yarn-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name, instance_id) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, } config_files.update(args.yarn_config.configuration.raw_files) return config_files
def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["hbase-site.xml"]) galaxy_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["galaxy-site.xml"]) zookeeper_properties = deploy_utils.generate_properties_file(args, args.fds_config.configuration.generated_files["zookeeper.properties"]) mapred_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["mapred-site.xml"]) yarn_site_xml = deploy_utils.generate_site_xml(args, args.fds_config.configuration.generated_files["yarn-site.xml"]) passport_properties = deploy_utils.generate_properties_file(args, args.fds_config.configuration.generated_files["passport.properties"]) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "galaxy-site.xml": galaxy_site_xml, "zookeeper.properties": zookeeper_properties, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "passport.properties": passport_properties, } config_files.update(args.fds_config.configuration.raw_files) return config_files
def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["hbase-site.xml"]) galaxy_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["galaxy-site.xml"]) zookeeper_properties = deploy_utils.generate_properties_file( args, args.fds_config.configuration.generated_files["zookeeper.properties"]) mapred_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["mapred-site.xml"]) yarn_site_xml = deploy_utils.generate_site_xml( args, args.fds_config.configuration.generated_files["yarn-site.xml"]) passport_properties = deploy_utils.generate_properties_file( args, args.fds_config.configuration.generated_files["passport.properties"]) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "galaxy-site.xml": galaxy_site_xml, "zookeeper.properties": zookeeper_properties, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "passport.properties": passport_properties, } config_files.update(args.fds_config.configuration.raw_files) return config_files
def generate_client_config(args, artifact, version): config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster, artifact, version) master_host = args.hbase_config.jobs["master"].hosts[0] config_path = "%s/%s/%s-%s/conf" % (args.package_root, args.cluster, artifact, version) deploy_utils.write_file("%s/hbase-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hbase-site.xml"])) deploy_utils.write_file("%s/hadoop-metrics.properties" % config_path, generate_metrics_config(args, master_host, "master")) deploy_utils.write_file("%s/core-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["core-site.xml"])) deploy_utils.write_file("%s/hdfs-site.xml" % config_path, deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hdfs-site.xml"])) deploy_utils.write_file("%s/jaas.conf" % config_path, deploy_zookeeper.generate_client_jaas_config(args)) deploy_utils.write_file("%s/krb5.conf" % config_path, args.hbase_config.configuration.raw_files["krb5.conf"]) update_hbase_env_sh(args, artifact, version)
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml(args, args.hbase_config.configuration.generated_files["hbase-site.xml"]) hadoop_metrics_properties = generate_metrics_config(args, host, job_name) zk_jaas_conf = generate_zk_jaas_config(args) hbase_raw_files = args.hbase_config.configuration.raw_files config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "hadoop-metrics.properties": hadoop_metrics_properties, "jaas.conf": zk_jaas_conf, "configuration.xsl": hbase_raw_files["configuration.xsl"], "log4j.xml": hbase_raw_files["log4j.xml"], "krb5.conf": hbase_raw_files["krb5.conf"], } return config_files
def generate_configs(args, host, job_name, instance_id): core_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hbase-site.xml"]) hadoop_metrics_properties = generate_metrics_config( args, host, job_name, instance_id) zk_jaas_conf = generate_zk_jaas_config(args) config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "hadoop-metrics.properties": hadoop_metrics_properties, "jaas.conf": zk_jaas_conf, } config_files.update(args.hbase_config.configuration.raw_files) return config_files
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hdfs-site.xml"]) hbase_site_xml = deploy_utils.generate_site_xml( args, args.hbase_config.configuration.generated_files["hbase-site.xml"]) hadoop_metrics_properties = generate_metrics_config(args, host, job_name) zk_jaas_conf = generate_zk_jaas_config(args) hbase_raw_files = args.hbase_config.configuration.raw_files config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "hbase-site.xml": hbase_site_xml, "hadoop-metrics.properties": hadoop_metrics_properties, "jaas.conf": zk_jaas_conf, "configuration.xsl": hbase_raw_files["configuration.xsl"], "log4j.xml": hbase_raw_files["log4j.xml"], "krb5.conf": hbase_raw_files["krb5.conf"], } return config_files
def generate_configs(args, host, job_name): core_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["core-site.xml"]) hdfs_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["hdfs-site.xml"]) mapred_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["mapred-site.xml"]) yarn_site_xml = deploy_utils.generate_site_xml(args, args.yarn_config.configuration.generated_files["yarn-site.xml"]) hadoop_metrics2_properties = generate_metrics_config(args, host, job_name) yarn_raw_files = args.yarn_config.configuration.raw_files config_files = { "core-site.xml": core_site_xml, "hdfs-site.xml": hdfs_site_xml, "mapred-site.xml": mapred_site_xml, "yarn-site.xml": yarn_site_xml, "hadoop-metrics2.properties": hadoop_metrics2_properties, "configuration.xsl": yarn_raw_files["configuration.xsl"], "log4j.xml": yarn_raw_files["log4j.xml"], "krb5.conf": yarn_raw_files["krb5.conf"], "excludes": yarn_raw_files["excludes"], } return config_files