def _install_es(self, gigabytes):
volumes = self.instance.markup.drives
if not fabric_files.exists("/usr/local/elasticsearch"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
with cd("/home/ec2-user/temp"):
run('rm -f '+JRE)
put("resources/"+JRE, JRE)
sudo("rpm -i "+JRE)
sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz')
run('tar zxfv elasticsearch-1.7.1.tar.gz')
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-1.7.1/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo('bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.7.1')
#REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE EBS VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount '+k.device)
sudo('yes | sudo mkfs -t ext4 '+k.device)
sudo('mkdir '+k.path)
sudo('sudo mount '+k.device+' '+k.path)
#ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root memlock unlimited' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user memlock unlimited' /etc/security/limits.conf")
# EFFECTIVE LOGIN TO LOAD CHANGES TO FILE HANDLES
# sudo("sudo -i -u ec2-user")
if not fabric_files.exists("/data1/logs"):
sudo('mkdir /data1/logs')
sudo('mkdir /data1/heapdump')
#INCREASE NUMBER OF FILE HANDLES
# sudo("sysctl -w fs.file-max=64000")
# COPY CONFIG FILE TO ES DIR
if not fabric_files.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
yml = File("./examples/config/es_config.yml").read().replace("\r", "")
yml = expand_template(yml, {
"id": Random.hex(length=8),
"data_paths": ",".join("/data"+unicode(i+1) for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
# FOR SOME REASON THE export COMMAND DOES NOT SEEM TO WORK
# THIS SCRIPT SETS THE ES_MIN_MEM/ES_MAX_MEM EXPLICITLY
if not fabric_files.exists("/usr/local/elasticsearch/bin/elasticsearch.in.sh"):
sh = File("./examples/config/es_run.sh").read().replace("\r", "")
sh = expand_template(sh, {"memory": unicode(int(gigabytes/2))})
File("./results/temp/elasticsearch.in.sh").write(sh)
with cd("/home/ec2-user"):
put("./results/temp/elasticsearch.in.sh", './temp/elasticsearch.in.sh', use_sudo=True)
sudo("cp -f ./temp/elasticsearch.in.sh /usr/local/elasticsearch/bin/elasticsearch.in.sh")
def _install_es(self, gigabytes, es_version="6.2.3"):
volumes = self.instance.markup.drives
if not fabric_files.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
response = run("java -version", warn_only=True)
if "Java(TM) SE Runtime Environment" not in response:
with cd("/home/ec2-user/temp"):
run('rm -f '+JRE)
put(LOCAL_JRE, JRE)
sudo("rpm -i "+JRE)
sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-'+es_version+'.tar.gz')
run('tar zxfv elasticsearch-'+es_version+'.tar.gz')
sudo("rm -fr /usr/local/elasticsearch", warn_only=True)
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-'+es_version+'/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount '+k.device)
sudo('yes | sudo mkfs -t ext4 '+k.device)
# ES AND JOURNALLING DO NOT MIX
sudo('tune2fs -o journal_data_writeback '+k.device)
sudo('tune2fs -O ^has_journal '+k.device)
sudo('mkdir '+k.path)
sudo('sudo mount '+k.device+' '+k.path)
sudo('chown -R ec2-user:ec2-user '+k.path)
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf")
sudo("sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf")
if not fabric_files.exists("/data1/logs"):
run('mkdir /data1/logs')
run('mkdir /data1/heapdump')
# COPY CONFIG FILES TO ES DIR
if not fabric_files.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
put("./examples/config/es6_log4j2.properties", '/usr/local/elasticsearch/config/log4j2.properties', use_sudo=True)
jvm = File("./examples/config/es6_jvm.options").read().replace('\r', '')
jvm = expand_template(jvm, {"memory": int(gigabytes/2)})
File("./results/temp/jvm.options").write(jvm)
put("./results/temp/jvm.options", '/usr/local/elasticsearch/config/jvm.options', use_sudo=True)
yml = File("./examples/config/es6_config.yml").read().replace("\r", "")
yml = expand_template(yml, {
"id": self.instance.ip_address,
"data_paths": ",".join("/data" + text_type(i + 1) for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")
def _install_es(self, gigabytes):
volumes = self.instance.markup.drives
if not fabric_files.exists("/usr/local/elasticsearch"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error(
"Expecting {{file}} on manager to spread to ES instances",
file=LOCAL_JRE)
with cd("/home/ec2-user/temp"):
run('rm -f ' + JRE)
put("resources/" + JRE, JRE)
sudo("rpm -i " + JRE)
sudo(
"alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000"
)
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz'
)
run('tar zxfv elasticsearch-1.7.1.tar.gz')
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-1.7.1/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo(
'bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.7.1'
)
#REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE EBS VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount ' + k.device)
sudo('yes | sudo mkfs -t ext4 ' + k.device)
sudo('mkdir ' + k.path)
sudo('sudo mount ' + k.device + ' ' + k.path)
#ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\" + k.device + " " + k.path +
" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf",
"./results/temp/sysctl.conf",
use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0",
"")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0",
"")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf",
"/etc/sysctl.conf",
use_sudo=True)
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 50000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root memlock unlimited' /etc/security/limits.conf")
sudo(
"sed -i '$ a\\ec2-user soft nofile 50000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user memlock unlimited' /etc/security/limits.conf"
)
# EFFECTIVE LOGIN TO LOAD CHANGES TO FILE HANDLES
# sudo("sudo -i -u ec2-user")
if not fabric_files.exists("/data1/logs"):
sudo('mkdir /data1/logs')
sudo('mkdir /data1/heapdump')
#INCREASE NUMBER OF FILE HANDLES
# sudo("sysctl -w fs.file-max=64000")
# COPY CONFIG FILE TO ES DIR
if not fabric_files.exists(
"/usr/local/elasticsearch/config/elasticsearch.yml"):
yml = File("./examples/config/es_config.yml").read().replace(
"\r", "")
yml = expand_template(
yml, {
"id":
Random.hex(length=8),
"data_paths":
",".join("/data" + unicode(i + 1)
for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml",
'/usr/local/elasticsearch/config/elasticsearch.yml',
use_sudo=True)
# FOR SOME REASON THE export COMMAND DOES NOT SEEM TO WORK
# THIS SCRIPT SETS THE ES_MIN_MEM/ES_MAX_MEM EXPLICITLY
if not fabric_files.exists(
"/usr/local/elasticsearch/bin/elasticsearch.in.sh"):
sh = File("./examples/config/es_run.sh").read().replace("\r", "")
sh = expand_template(sh, {"memory": unicode(int(gigabytes / 2))})
File("./results/temp/elasticsearch.in.sh").write(sh)
with cd("/home/ec2-user"):
put("./results/temp/elasticsearch.in.sh",
'./temp/elasticsearch.in.sh',
use_sudo=True)
sudo(
"cp -f ./temp/elasticsearch.in.sh /usr/local/elasticsearch/bin/elasticsearch.in.sh"
)
def _install_es(self, gigabytes, es_version="6.2.3"):
volumes = self.instance.markup.drives
if not fabric_files.exists(
"/usr/local/elasticsearch/config/elasticsearch.yml"):
with cd("/home/ec2-user/"):
run("mkdir -p temp")
if not File(LOCAL_JRE).exists:
Log.error(
"Expecting {{file}} on manager to spread to ES instances",
file=LOCAL_JRE)
response = run("java -version", warn_only=True)
if "Java(TM) SE Runtime Environment" not in response:
with cd("/home/ec2-user/temp"):
run('rm -f ' + JRE)
put(LOCAL_JRE, JRE)
sudo("rpm -i " + JRE)
sudo(
"alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000"
)
run("export JAVA_HOME=/usr/java/default")
with cd("/home/ec2-user/"):
run('wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-'
+ es_version + '.tar.gz')
run('tar zxfv elasticsearch-' + es_version + '.tar.gz')
sudo("rm -fr /usr/local/elasticsearch", warn_only=True)
sudo('mkdir /usr/local/elasticsearch')
sudo('cp -R elasticsearch-' + es_version +
'/* /usr/local/elasticsearch/')
with cd('/usr/local/elasticsearch/'):
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
# https://github.com/elasticsearch/elasticsearch-cloud-aws
sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
self.conn = self.instance.connection
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
for i, k in enumerate(volumes):
if not fabric_files.exists(k.path):
with fabric_settings(warn_only=True):
sudo('sudo umount ' + k.device)
sudo('yes | sudo mkfs -t ext4 ' + k.device)
# ES AND JOURNALLING DO NOT MIX
sudo('tune2fs -o journal_data_writeback ' + k.device)
sudo('tune2fs -O ^has_journal ' + k.device)
sudo('mkdir ' + k.path)
sudo('sudo mount ' + k.device + ' ' + k.path)
sudo('chown -R ec2-user:ec2-user ' + k.path)
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
sudo("sed -i '$ a\\" + k.device + " " + k.path +
" ext4 defaults,nofail 0 2' /etc/fstab")
# TEST IT IS WORKING
sudo('mount -a')
# INCREASE THE FILE HANDLE LIMITS
with cd("/home/ec2-user/"):
File("./results/temp/sysctl.conf").delete()
get("/etc/sysctl.conf",
"./results/temp/sysctl.conf",
use_sudo=True)
lines = File("./results/temp/sysctl.conf").read()
if lines.find("fs.file-max = 100000") == -1:
lines += "\nfs.file-max = 100000"
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0",
"")
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0",
"")
File("./results/temp/sysctl.conf").write(lines)
put("./results/temp/sysctl.conf",
"/etc/sysctl.conf",
use_sudo=True)
sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
sudo("sysctl -p")
# INCREASE FILE HANDLE PERMISSIONS
sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
sudo(
"sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf"
)
sudo(
"sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf"
)
if not fabric_files.exists("/data1/logs"):
run('mkdir /data1/logs')
run('mkdir /data1/heapdump')
# COPY CONFIG FILES TO ES DIR
if not fabric_files.exists(
"/usr/local/elasticsearch/config/elasticsearch.yml"):
put("./examples/config/es6_log4j2.properties",
'/usr/local/elasticsearch/config/log4j2.properties',
use_sudo=True)
jvm = File("./examples/config/es6_jvm.options").read().replace(
'\r', '')
jvm = expand_template(jvm, {"memory": int(gigabytes / 2)})
File("./results/temp/jvm.options").write(jvm)
put("./results/temp/jvm.options",
'/usr/local/elasticsearch/config/jvm.options',
use_sudo=True)
yml = File("./examples/config/es6_config.yml").read().replace(
"\r", "")
yml = expand_template(
yml, {
"id":
self.instance.ip_address,
"data_paths":
",".join("/data" + text_type(i + 1)
for i, _ in enumerate(volumes))
})
File("./results/temp/elasticsearch.yml").write(yml)
put("./results/temp/elasticsearch.yml",
'/usr/local/elasticsearch/config/elasticsearch.yml',
use_sudo=True)
sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")