def install_hadoop():
path = os.path.dirname(os.path.realpath(__file__))
config = json.load(open(path+'/cluster-config.json'))
if 'hadoop' not in subprocess.check_output(['ls']).split('\n'):
p = subprocess.Popen(['wget', 'http://mirror.netinch.com/pub/apache/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz'])
if 0 == p.wait():
# exact
p = subprocess.Popen(['tar', '-zvxf', 'hadoop-2.6.0.tar.gz'])
p.wait()
subprocess.call(['mv', 'hadoop-2.6.0', 'hadoop'])
# cp hadoop
subprocess.call(['sudo', 'rm', '-rf', '/usr/local/hadoop'])
subprocess.call(['sudo', 'cp', '-r', 'hadoop', '/usr/local/hadoop'])
subprocess.call(['sudo', 'chown', '-R', 'cloud-user', '/usr/local/hadoop'])
# conf
subprocess.call(['rm', '-rf', '/usr/local/hadoop/etc'])
subprocess.call(['cp', '-r', path+'/hadoop/etc', '/usr/local/hadoop/etc'])
# mkdir hadoop tmp dir
if 'hadoop' not in subprocess.check_output(['ls', '/mnt']).split('\n'):
subprocess.call(['sudo', 'mkdir', '-p', '/mnt/hadoop'])
if 'namenode' not in subprocess.check_output(['ls', '/mnt/hadoop']).split('\n'):
if 0 == subprocess.call(['sudo', 'mkdir', '-p', '/mnt/hadoop/namenode']):
print("/mnt/hadoop/namenode created successfully")
if 'datanode' not in subprocess.check_output(['ls', '/mnt/hadoop']).split('\n'):
if 0 == subprocess.call(['sudo', 'mkdir', '-p', '/mnt/hadoop/datanode']):
print("/mnt/hadoop/datanode created successfully")
subprocess.call(['sudo', 'chown', '-R', 'cloud-user', '/mnt/hadoop'])
# set hadoop home
appendline('/home/cloud-user/.profile', 'export HADOOP_HOME=/usr/local/hadoop/')
appendline('/home/cloud-user/.profile', 'export PATH=$HADOOP_HOME/bin:$PATH')
def install_zookeeper():
# /mnt/zookeeper/data chown
config = json.load(open('zookeeper-config.json'));
data_dir_maked = subprocess.check_call(["sudo", "mkdir", "-p", "/mnt/zookeeper/data"])
if 0 == data_dir_maked:
subprocess.call(["sudo", "chown", "-R", "cloud-user", "/mnt/zookeeper"])
else:
print("Create dirctory /mnt/zookeeper/data failed")
sys.exist(1)
print("Create dirctory /mnt/zookeeper/data successfully")
# myid
myip = get_ip_address()
mynode = [node for node in config['nodes'] if node['ip'] == myip][0]
open("/mnt/zookeeper/data/myid", "w").write(str(mynode['id']))
print("Set myid for zookeeper successfully")
# cp zookeeper
subprocess.call(['sudo', 'rm', '-rf', '/usr/local/zookeeper'])
subprocess.call(['sudo', 'cp', '-r', './zookeeper', '/usr/local/zookeeper'])
for node in config['nodes']:
appendline('/usr/local/zookeeper/conf/zoo.cfg', 'server.'+str(node['id'])+'=zoo'+str(node['id'])+':2888:3888')
subprocess.call(['sudo', 'chown', '-R', 'cloud-user', '/usr/local/zookeeper'])
# hosts
for node in config['nodes']:
appendline('/etc/hosts', node['ip']+'\t'+'zoo'+str(node['id']))
def install_zookeeper():
# /mnt/zookeeper/data chown
config = json.load(open('zookeeper-config.json'))
data_dir_maked = subprocess.check_call(
["sudo", "mkdir", "-p", "/mnt/zookeeper/data"])
if 0 == data_dir_maked:
subprocess.call(
["sudo", "chown", "-R", "cloud-user", "/mnt/zookeeper"])
else:
print("Create dirctory /mnt/zookeeper/data failed")
sys.exist(1)
print("Create dirctory /mnt/zookeeper/data successfully")
# myid
myip = get_ip_address()
mynode = [node for node in config['nodes'] if node['ip'] == myip][0]
open("/mnt/zookeeper/data/myid", "w").write(str(mynode['id']))
print("Set myid for zookeeper successfully")
# cp zookeeper
subprocess.call(['sudo', 'rm', '-rf', '/usr/local/zookeeper'])
subprocess.call(
['sudo', 'cp', '-r', './zookeeper', '/usr/local/zookeeper'])
for node in config['nodes']:
appendline(
'/usr/local/zookeeper/conf/zoo.cfg', 'server.' + str(node['id']) +
'=zoo' + str(node['id']) + ':2888:3888')
subprocess.call(
['sudo', 'chown', '-R', 'cloud-user', '/usr/local/zookeeper'])
# hosts
for node in config['nodes']:
appendline('/etc/hosts', node['ip'] + '\t' + 'zoo' + str(node['id']))
def install_jdk7():
if 'jvm' not in subprocess.check_output(['ls', '/usr/lib']).split('\n') \
or 'java-1.7.0-openjdk-amd64' not in subprocess.check_output(['ls', '/usr/lib/jvm']).split('\n'):
update = subprocess.Popen(["sudo", "apt-get", "update"])
if update.wait() == 0:
jdk_installed = subprocess.check_call(["sudo", "apt-get", "install", "-y", "openjdk-7-jdk"])
if jdk_installed == 0:
print("JDK 1.7 installed successfully")
else:
print("apt-get update failed on server")
else:
print("JDK 1.7 is already installed.")
appendline('/home/cloud-user/.profile', 'export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64/')
appendline('/home/cloud-user/.profile', 'export PATH=$JAVA_HOME/bin:$PATH')
def install_jdk7():
if 'jvm' not in subprocess.check_output(['ls', '/usr/lib']).split('\n') \
or 'java-1.7.0-openjdk-amd64' not in subprocess.check_output(['ls', '/usr/lib/jvm']).split('\n'):
update = subprocess.Popen(["sudo", "apt-get", "update"])
if update.wait() == 0:
jdk_installed = subprocess.check_call(
["sudo", "apt-get", "install", "-y", "openjdk-7-jdk"])
if jdk_installed == 0:
print("JDK 1.7 installed successfully")
else:
print("apt-get update failed on server")
else:
print("JDK 1.7 is already installed.")
appendline('/home/cloud-user/.profile',
'export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64/')
appendline('/home/cloud-user/.profile', 'export PATH=$JAVA_HOME/bin:$PATH')
def install_hadoop():
path = os.path.dirname(os.path.realpath(__file__))
config = json.load(open(path + '/cluster-config.json'))
if 'hadoop' not in subprocess.check_output(['ls']).split('\n'):
p = subprocess.Popen([
'wget',
'http://mirror.netinch.com/pub/apache/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz'
])
if 0 == p.wait():
# exact
p = subprocess.Popen(['tar', '-zvxf', 'hadoop-2.6.0.tar.gz'])
p.wait()
subprocess.call(['mv', 'hadoop-2.6.0', 'hadoop'])
# cp hadoop
subprocess.call(['sudo', 'rm', '-rf', '/usr/local/hadoop'])
subprocess.call(['sudo', 'cp', '-r', 'hadoop', '/usr/local/hadoop'])
subprocess.call(['sudo', 'chown', '-R', 'cloud-user', '/usr/local/hadoop'])
# conf
subprocess.call(['rm', '-rf', '/usr/local/hadoop/etc'])
subprocess.call(
['cp', '-r', path + '/hadoop/etc', '/usr/local/hadoop/etc'])
# mkdir hadoop tmp dir
if 'hadoop' not in subprocess.check_output(['ls', '/mnt']).split('\n'):
subprocess.call(['sudo', 'mkdir', '-p', '/mnt/hadoop'])
if 'namenode' not in subprocess.check_output(['ls', '/mnt/hadoop'
]).split('\n'):
if 0 == subprocess.call(
['sudo', 'mkdir', '-p', '/mnt/hadoop/namenode']):
print("/mnt/hadoop/namenode created successfully")
if 'datanode' not in subprocess.check_output(['ls', '/mnt/hadoop'
]).split('\n'):
if 0 == subprocess.call(
['sudo', 'mkdir', '-p', '/mnt/hadoop/datanode']):
print("/mnt/hadoop/datanode created successfully")
subprocess.call(['sudo', 'chown', '-R', 'cloud-user', '/mnt/hadoop'])
# set hadoop home
appendline('/home/cloud-user/.profile',
'export HADOOP_HOME=/usr/local/hadoop/')
appendline('/home/cloud-user/.profile',
'export PATH=$HADOOP_HOME/bin:$PATH')
#!/bin/python
from __future__ import print_function
import subprocess
import sys
import os
import json
from util import appendline, get_ip_address
if __name__ == "__main__":
path = os.path.dirname(os.path.realpath(__file__))
config = json.load(open(path+'/cluster-config.json'));
for node in config['nodes']:
appendline('/etc/hosts', node['ip']+'\t'+node['host'] + '\t'+node['hostname'])
appendline('/etc/hosts', '192.168.1.25\t'+'zoo1')
#!/bin/python
from __future__ import print_function
import subprocess
import sys
import os
import json
from util import appendline, get_ip_address
if __name__ == "__main__":
path = os.path.dirname(os.path.realpath(__file__))
config = json.load(open(path + '/cluster-config.json'))
for node in config['nodes']:
appendline('/etc/hosts',
node['ip'] + '\t' + node['host'] + '\t' + node['hostname'])
appendline('/etc/hosts', '192.168.1.25\t' + 'zoo1')