def setHdfsFilev2(interval): #file = xmlparsing('/H2/etc/hadoop/hdfs-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/hdfs-site.xml') # setting datanode directory file.appendProperty('dfs.data.dir', 'file:/data/datanodeData', 'Directory for saving the datanode data') file.appendProperty('dfs.heartbeat.interval',interval,'heartbeat time interval') file.saveFile()
def setHdfsFilev1(nnip,nnwebport): file = xmlparsing('/etc/hadoop/hdfs-site.xml') # setting namenode directory file.appendProperty('dfs.name.dir','/data/namenodeData,/data/backup') # setting web(http) management portal file.appendProperty('dfs.http.address',nnip+":"+nnwebport,'web management port.') file.saveFile()
def setCoreFilev2(nnip, nnport): file = xmlparsing('/H2/etc/hadoop/core-site.xml') #file = xmlparsing(os.getenv('HADOOP_HOME')+'/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name', 'hdfs://' + nnip + ':' + nnport, 'namenode address') # file.appendProperty('hadoop.tmp.dir','/tmp/hadoop',' temporary data') file.saveFile()
def setHdfsFilev2(nnip,nnwebport): #file = xmlparsing('/H2/etc/hadoop/hdfs-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME')+'/etc/hadoop/hdfs-site.xml') # setting namenode directory file.appendProperty('dfs.name.dir','file:/data/namenodeData,file:/data/backup') # setting web(http) management portal file.appendProperty('dfs.http.address',nnip+":"+nnwebport,'web management port.') file.saveFile()
def setYarnFile(rmip): #file = xmlparsing('/H2/etc/hadoop/yarn-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/yarn-site.xml') file.appendProperty('yarn.resourcemanager.resource-tracker.address', rmip + ':8025') file.appendProperty('yarn.resourcemanager.scheduler.address', rmip + ':8030') file.saveFile()
def setHdfsFilev1(nnip, nnwebport): file = xmlparsing('/etc/hadoop/hdfs-site.xml') # setting namenode directory file.appendProperty('dfs.name.dir', '/data/namenodeData,/data/backup') # setting web(http) management portal file.appendProperty('dfs.http.address', nnip + ":" + nnwebport, 'web management port.') file.saveFile()
def setHdfsFilev1(checkperiod): file = xmlparsing('/etc/hadoop/hdfs-site.xml') file.appendProperty('dfs.name.dir','/data/namenodeData') # file.appendProperty('dfs.http.address',nnip+':'+nnwebport) # file.appendProperty('dfs.secondary.http.address',snnip+':'+snnport) file.appendProperty('fs.checkpoint.dir','/data/check') file.appendProperty('fs.checkpoint.edits.dir','/data/edits') file.appendProperty('fs.checkpoint.period',checkperiod) file.saveFile()
def setHdfsFilev1(checkperiod): file = xmlparsing('/etc/hadoop/hdfs-site.xml') file.appendProperty('dfs.name.dir', '/data/namenodeData') # file.appendProperty('dfs.http.address',nnip+':'+nnwebport) # file.appendProperty('dfs.secondary.http.address',snnip+':'+snnport) file.appendProperty('fs.checkpoint.dir', '/data/check') file.appendProperty('fs.checkpoint.edits.dir', '/data/edits') file.appendProperty('fs.checkpoint.period', checkperiod) file.saveFile()
def setHdfsFilev2(nnip, nnwebport): #file = xmlparsing('/H2/etc/hadoop/hdfs-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/hdfs-site.xml') # setting namenode directory file.appendProperty('dfs.name.dir', 'file:/data/namenodeData,file:/data/backup') # setting web(http) management portal file.appendProperty('dfs.http.address', nnip + ":" + nnwebport, 'web management port.') file.saveFile()
def extrainstall(rmip): os.chdir('/root/HadoopInstall/') print commands.getoutput('cp -f '+os.getenv('HADOOP_HOME')+'/etc/hadoop/mapred-site.xml.template '+os.getenv('HADOOP_HOME')+'/etc/hadoop/mapred-site.xml') # Installing splunk commands.getoutput('rpm -i '+packagename('splunk')) status,output = commands.getstatusoutput('/opt/splunk/bin/splunk | grep "Splunk web interface" ') splunkaddress = None if status == 0: splunkaddress=output[ output.rfind(' ') :] # Installing pig framework commands.getoutput('tar -xzf '+packagename('pig')) print commands.getoutput('mv -f pig-0.12.1 /pig') if commands.getoutput('cat /root/.bashrc | grep PIG_HOME ') == '': file = open('/root/.bashrc','a') file.write('export PIG_HOME=/pig \n') file.write('export PATH=$PIG_HOME/bin:$PATH\n') file.close() commands.getoutput('chcon --reference /root/.bash_profile /root/.bashrc') file = None if hadoopversion == '1': file = xmlparsing('/etc/hadoop/mapred-site.xml') elif hadoopversion == '2': file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/mapred-site.xml') file.appendProperty('mapreduce.jobhistory.address', rmip + ':10020') file.saveFile() # Installing Hive framework commands.getoutput('tar -xzf ' + packagename('hive')) print commands.getoutput('mv -f apache-hive-1.2.1-bin /hive') if commands.getoutput('cat /root/.bashrc | grep HIVE_HOME ') == '': file = open('/root/.bashrc', 'a') file.write('export HIVE_HOME=/hive \n') file.write('export PATH=$HIVE_HOME/bin:$PATH \n') if hadoopversion=='2': file.write('export HADOOP_USER_CLASSPATH_FIRST=true') file.close() commands.getoutput(' chmod +x /root/.bashrc ; source /root/.bashrc') commands.getoutput('chcon --reference /root/.bash_profile /root/.bashrc')
def setCoreFilev1(nnip,nnport): file = xmlparsing('/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name','hdfs://'+nnip+':'+nnport,' namenode address') file.saveFile()
def setCoreFilev1(nnip,nnport): file = xmlparsing('/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name','hdfs://'+nnip+':'+nnport,' namenode address') #file.appendProperty('hadoop.tmp.dir','/usr/local/hadoop/tmp',' temporary data') file.saveFile()
def setMapredFilev2(framework): #commands.getoutput('mv '+os.getenv('HADOOP_HOME')+'/etc/hadoop/mapred-site.xml.template ' \ # + os.getenv('HADOOP_HOME')+'/etc/hadoop/mapred-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME') +'/etc/hadoop/mapred-site.xml') file.appendProperty('mapreduce.framework.name',framework) file.saveFile()
def setYarnFile(rmip): #file = xmlparsing('/H2/etc/hadoop/yarn-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME')+'/etc/hadoop/yarn-site.xml') file.appendProperty('yarn.resourcemanager.resource-tracker.address',rmip+':8025') file.appendProperty('yarn.resourcemanager.scheduler.address', rmip + ':8030') file.saveFile()
def setMapredFilev1(jbip,jbport): file = xmlparsing('/etc/hadoop/mapred-site.xml') # setting job tracker ip file.appendProperty('mapred.job.tracker',jbip+':'+jbport) file.saveFile()
def setHdfsFilev1(interval): file = xmlparsing('/etc/hadoop/hdfs-site.xml') # setting datanode directory file.appendProperty('dfs.data.dir', '/data/datanodeData', 'Directory for saving the datanode data') file.appendProperty('dfs.heartbeat.interval',interval,'heartbeat time interval') file.saveFile()
def setYarnFile(rmip): #file = xmlparsing('/H2/etc/hadoop/yarn-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME')+'/etc/hadoop/yarn-site.xml') file.appendProperty('yarn.resourcemanager.resource-tracker.address',rmip+':8025') file.appendProperty('yarn.nodemanager.aux-services','mapreduce_shuffle') file.saveFile()
def setCoreFilev2(nnip,nnport): #file = xmlparsing('/H2/etc/hadoop/core-site.xml') file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name','hdfs://'+nnip+':'+nnport,' namenode address') file.saveFile()
def setHdfsFilev1(replica,size): file = xmlparsing('/etc/hadoop/hdfs-site.xml') # setting datanode directory file.appendProperty('dfs.replication',replica,'Number of replications') file.appendProperty('dfs.block.size', size , 'Number of replications') file.saveFile()
def setCoreFilev1(nnip,nnport): file = xmlparsing(filename='/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name','hdfs://'+nnip+':'+nnport,' namenode address') #file.appendProperty('hadoop.tmp.dir','/usr/local/hadoop/tmp',' temporary data') file.saveFile()
def setCoreFilev1(nnip, nnport): file = xmlparsing('/etc/hadoop/core-site.xml') file.appendProperty('fs.default.name', 'hdfs://' + nnip + ':' + nnport, ' namenode address') file.saveFile()
def setHdfsFilev2(replica,size): file = xmlparsing(os.getenv('HADOOP_HOME') + '/etc/hadoop/hdfs-site.xml') file.appendProperty('dfs.replication', replica, 'Number of replications') file.appendProperty('dfs.block.size', size, 'Number of replications') file.saveFile()