def __init__(self,
hadoop_artifact_url,
hadoop_install_dir,
hadoop_data_dir,
template_conf_dir,
hostname='localhost',
secure_hadoop=False):
HadoopUtil.__init__(self, hadoop_artifact_url, hadoop_install_dir,
hadoop_data_dir, hostname)
self.rpmutil = RPMUtil()
self.hostname = hostname
self.hadoop_artifact_url = hadoop_artifact_url
self.hadoop_install_dir = hadoop_install_dir
self.hadoop_binary_loc = ''
self.hadoop_data_dir = hadoop_data_dir
self.template_conf_dir = template_conf_dir
self.secure_hadoop = secure_hadoop
# Constants
# under the hadoop template configuration directory
# both the below directories should be present
self.SECURE_DIR_NAME = "conf.secure" # secure configuration files location
self.NON_SECURE_DIR_NAME = "conf.pseudo" # non-secure configuration files location
self.DEPENDENCY_PKGS = [
"fuse-", # eg. fuse-2.8.3-4.el6.x86_64
"fuse-libs", # eg. fuse-libs-2.8.3-4.el6.x86_6
"nc-" # eg. 1.84-22.el6.x86_64"
]
self.HADOOP_UTILITY_RPMS = "utility/rpm/"
self.ZOOKEEPER_RPMS = "zookeeper/rpm/"
self.HADOOP_RPMS = "hadoop/rpm/"
self.HADOOP_ENVS = {
"HADOOP_HOME": "/usr/lib/gphd/hadoop/",
"HADOOP_COMMON_HOME": "/usr/lib/gphd/hadoop/",
"HADOOP_HDFS_HOME": "/usr/lib/gphd/hadoop-hdfs/",
"HADOOP_MAPRED_HOME": "/usr/lib/gphd/hadoop-mapreduce/",
"YARN_HOME": "/usr/lib/gphd/hadoop-yarn/",
"HADOOP_TMP_DIR": "%s/hadoop-hdfs/cache/" % self.hadoop_data_dir,
"MAPRED_TMP_DIR":
"%s/hadoop-mapreduce/cache/" % self.hadoop_data_dir,
"YARN_TMP_DIR": "%s/hadoop-yarn/cache/" % self.hadoop_data_dir,
"HADOOP_CONF_DIR": "/etc/hadoop/conf",
"HADOOP_LOG_DIR":
"%s/hadoop-logs/hadoop-hdfs" % self.hadoop_data_dir,
"MAPRED_LOG_DIR":
"%s/hadoop-logs/hadoop-mapreduce" % self.hadoop_data_dir,
"YARN_LOG_DIR": "%s/hadoop-logs/hadoop-yarn" % self.hadoop_data_dir
}
self.PKGS_TO_REMOVE = "^hadoop-*|^bigtop-*|^zookeeper-*|^parquet-*"
def __init__(
self, hadoop_src_artifact_url, hadoop_install_dir, hadoop_data_dir, template_conf_dir, hostname = 'localhost',
secure_hadoop = False
):
HadoopUtil.__init__(self, hadoop_src_artifact_url, hadoop_install_dir, hadoop_data_dir, hostname)
self.rpmutil = RPMUtil()
self.hostname = hostname
# we build the hadoop binaries from the source code so as to avoid any issues
self.hadoop_src_artifact_url = hadoop_src_artifact_url
self.hadoop_install_dir = hadoop_install_dir
self.hadoop_binary_loc = ''
self.template_conf_dir = template_conf_dir
self.secure_hadoop = secure_hadoop
# Constants
# under the hadoop template configuration directory
# both the below directories should be present
self.SECURE_DIR_NAME = "conf.secure" # secure configuration files location
self.NON_SECURE_DIR_NAME = "conf.pseudo" # non-secure configuration files location
def __init__(
self, hadoop_artifact_url, hadoop_install_dir, hadoop_data_dir, template_conf_dir, hostname = 'localhost',
secure_hadoop = False
):
HadoopUtil.__init__(self, hadoop_artifact_url, hadoop_install_dir, hadoop_data_dir, hostname)
self.rpmutil = RPMUtil()
self.hostname = hostname
self.hadoop_artifact_url = hadoop_artifact_url
self.hadoop_install_dir = hadoop_install_dir
self.hadoop_binary_loc = ''
self.hadoop_data_dir = hadoop_data_dir
self.template_conf_dir = template_conf_dir
self.secure_hadoop = secure_hadoop
# Constants
# under the hadoop template configuration directory
# both the below directories should be present
self.SECURE_DIR_NAME = "conf.secure" # secure configuration files location
self.NON_SECURE_DIR_NAME = "conf.pseudo" # non-secure configuration files location
self.DEPENDENCY_PKGS = [
"fuse-", # eg. fuse-2.8.3-4.el6.x86_64
"fuse-libs", # eg. fuse-libs-2.8.3-4.el6.x86_6
"nc-" # eg. 1.84-22.el6.x86_64"
]
self.HADOOP_UTILITY_RPMS = "utility/rpm/"
self.ZOOKEEPER_RPMS = "zookeeper/rpm/"
self.HADOOP_RPMS = "hadoop/rpm/"
self.HADOOP_ENVS = {
"HADOOP_HOME" : "/usr/lib/gphd/hadoop/",
"HADOOP_COMMON_HOME" : "/usr/lib/gphd/hadoop/",
"HADOOP_HDFS_HOME" : "/usr/lib/gphd/hadoop-hdfs/",
"HADOOP_MAPRED_HOME" : "/usr/lib/gphd/hadoop-mapreduce/",
"YARN_HOME" : "/usr/lib/gphd/hadoop-yarn/",
"HADOOP_TMP_DIR" : "%s/hadoop-hdfs/cache/" %self.hadoop_data_dir,
"MAPRED_TMP_DIR" : "%s/hadoop-mapreduce/cache/" %self.hadoop_data_dir,
"YARN_TMP_DIR" : "%s/hadoop-yarn/cache/" %self.hadoop_data_dir,
"HADOOP_CONF_DIR" : "/etc/hadoop/conf",
"HADOOP_LOG_DIR" : "%s/hadoop-logs/hadoop-hdfs" %self.hadoop_data_dir,
"MAPRED_LOG_DIR" : "%s/hadoop-logs/hadoop-mapreduce" %self.hadoop_data_dir,
"YARN_LOG_DIR" : "%s/hadoop-logs/hadoop-yarn" %self.hadoop_data_dir
}
self.PKGS_TO_REMOVE = "^hadoop-*|^bigtop-*|^zookeeper-*|^parquet-*"
def __init__(self,
hadoop_src_artifact_url,
hadoop_install_dir,
hadoop_data_dir,
template_conf_dir,
hostname='localhost',
secure_hadoop=False):
HadoopUtil.__init__(self, hadoop_src_artifact_url, hadoop_install_dir,
hadoop_data_dir, hostname)
self.rpmutil = RPMUtil()
self.hostname = hostname
# we build the hadoop binaries from the source code so as to avoid any issues
self.hadoop_src_artifact_url = hadoop_src_artifact_url
self.hadoop_install_dir = hadoop_install_dir
self.hadoop_binary_loc = ''
self.template_conf_dir = template_conf_dir
self.secure_hadoop = secure_hadoop
# Constants
# under the hadoop template configuration directory
# both the below directories should be present
self.SECURE_DIR_NAME = "conf.secure" # secure configuration files location
self.NON_SECURE_DIR_NAME = "conf.pseudo" # non-secure configuration files location
