def configure_hdfs_base(self, clustername, namenodes, port, webhdfs_port): dc = self.hadoop_base.dist_config core_site = dc.path('hadoop_conf') / 'core-site.xml' with utils.xmlpropmap_edit_in_place(core_site) as props: props['hadoop.proxyuser.hue.hosts'] = "*" props['hadoop.proxyuser.hue.groups'] = "*" props['hadoop.proxyuser.oozie.groups'] = '*' props['hadoop.proxyuser.oozie.hosts'] = '*' if 'lzo' in self.hadoop_base.resources: props['io.compression.codecs'] = ( 'org.apache.hadoop.io.compress.GzipCodec, ' 'org.apache.hadoop.io.compress.DefaultCodec, ' 'org.apache.hadoop.io.compress.BZip2Codec, ' 'org.apache.hadoop.io.compress.SnappyCodec, ' 'com.hadoop.compression.lzo.LzoCodec, ' 'com.hadoop.compression.lzo.LzopCodec') props[ 'io.compression.codec.lzo.class'] = 'com.hadoop.compression.lzo.LzoCodec' else: props['io.compression.codecs'] = ( 'org.apache.hadoop.io.compress.GzipCodec, ' 'org.apache.hadoop.io.compress.DefaultCodec, ' 'org.apache.hadoop.io.compress.BZip2Codec, ' 'org.apache.hadoop.io.compress.SnappyCodec') props['fs.defaultFS'] = "hdfs://{clustername}".format( clustername=clustername, port=port) hdfs_site = dc.path('hadoop_conf') / 'hdfs-site.xml' with utils.xmlpropmap_edit_in_place(hdfs_site) as props: props['dfs.webhdfs.enabled'] = "true" props['dfs.namenode.name.dir'] = dc.path( 'hdfs_dir_base') / 'cache/hadoop/dfs/name' props['dfs.datanode.data.dir'] = dc.path( 'hdfs_dir_base') / 'cache/hadoop/dfs/name' props[ 'dfs.permissions'] = 'false' # TODO - secure this hadoop installation! props['dfs.nameservices'] = clustername props['dfs.client.failover.proxy.provider.%s' % clustername] = \ 'org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider' props['dfs.ha.fencing.methods'] = 'sshfence\nshell(/bin/true)' props['dfs.ha.fencing.ssh.private-key-files'] = utils.ssh_priv_key( 'hdfs') props['dfs.ha.namenodes.%s' % clustername] = ','.join(namenodes) for node in namenodes: props['dfs.namenode.rpc-address.%s.%s' % (clustername, node)] = '%s:%s' % (node, port) props['dfs.namenode.http-address.%s.%s' % (clustername, node)] = '%s:%s' % (node, webhdfs_port)
def configure_hdfs_base(self, clustername, namenodes, port, webhdfs_port): dc = self.hadoop_base.dist_config core_site = dc.path('hadoop_conf') / 'core-site.xml' with utils.xmlpropmap_edit_in_place(core_site) as props: props['hadoop.proxyuser.hue.hosts'] = "*" props['hadoop.proxyuser.hue.groups'] = "*" props['hadoop.proxyuser.oozie.groups'] = '*' props['hadoop.proxyuser.oozie.hosts'] = '*' if 'lzo' in self.hadoop_base.resources: props['io.compression.codecs'] = ('org.apache.hadoop.io.compress.GzipCodec, ' 'org.apache.hadoop.io.compress.DefaultCodec, ' 'org.apache.hadoop.io.compress.BZip2Codec, ' 'org.apache.hadoop.io.compress.SnappyCodec, ' 'com.hadoop.compression.lzo.LzoCodec, ' 'com.hadoop.compression.lzo.LzopCodec') props['io.compression.codec.lzo.class'] = 'com.hadoop.compression.lzo.LzoCodec' else: props['io.compression.codecs'] = ('org.apache.hadoop.io.compress.GzipCodec, ' 'org.apache.hadoop.io.compress.DefaultCodec, ' 'org.apache.hadoop.io.compress.BZip2Codec, ' 'org.apache.hadoop.io.compress.SnappyCodec') props['fs.defaultFS'] = "hdfs://{clustername}".format(clustername=clustername, port=port) hdfs_site = dc.path('hadoop_conf') / 'hdfs-site.xml' with utils.xmlpropmap_edit_in_place(hdfs_site) as props: props['dfs.webhdfs.enabled'] = "true" props['dfs.namenode.name.dir'] = dc.path('hdfs_dir_base') / 'cache/hadoop/dfs/name' props['dfs.datanode.data.dir'] = dc.path('hdfs_dir_base') / 'cache/hadoop/dfs/name' props['dfs.permissions'] = 'false' # TODO - secure this hadoop installation! props['dfs.nameservices'] = clustername props['dfs.client.failover.proxy.provider.%s' % clustername] = \ 'org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider' props['dfs.ha.fencing.methods'] = 'sshfence\nshell(/bin/true)' props['dfs.ha.fencing.ssh.private-key-files'] = utils.ssh_priv_key('hdfs') props['dfs.ha.namenodes.%s' % clustername] = ','.join(namenodes) for node in namenodes: props['dfs.namenode.rpc-address.%s.%s' % (clustername, node)] = '%s:%s' % (node, port) props['dfs.namenode.http-address.%s.%s' % (clustername, node)] = '%s:%s' % (node, webhdfs_port)
def generate_ssh_key(): utils.generate_ssh_key('hdfs') leadership.leader_set({ 'ssh-key-priv': utils.ssh_priv_key('hdfs').text(), 'ssh-key-pub': utils.ssh_pub_key('hdfs').text(), })