def install_mongo(self):
"""
installs mongodb on the system
"""
self.logger.info('entering install_mongo method')
if not linux_utils.check_root():
self.logger.fatal("Please run this as root")
import_key = 'sudo apt-key adv --keyserver ' \
'hkp://keyserver.ubuntu.com:80 --recv ' \
'9DA31620334BD75D9DCB49F368818C72E52529D4'
self.executor.execute(import_key)
create_list = 'echo "deb [ arch=amd64 ] https://repo.mongodb.org/' \
'apt/ubuntu bionic/mongodb-org/4.0 multiverse" | ' \
'sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list'
self.executor.execute(create_list)
reload_packages = 'sudo apt-get update'
self.executor.execute(reload_packages)
self.logger.debug('installing mongo')
install_mongo = 'sudo apt-get install -y mongodb-org'
self.executor.execute(install_mongo)
hold = """echo "mongodb-org hold" | sudo dpkg --set-selections
echo "mongodb-org-server hold" | sudo dpkg --set-selections
echo "mongodb-org-shell hold" | sudo dpkg --set-selections
echo "mongodb-org-mongos hold" | sudo dpkg --set-selections
echo "mongodb-org-tools hold" | sudo dpkg --set-selections"""
self.executor.execute(hold)
self.logger.info('exiting install_mongo')
def install(self):
if not check_root():
print("Please run the program using sudo privileges")
raise PermissionDeniedException(
"Permission Denied. Run with admin rights.")
self.setup_logstash()
def execute(self):
"""
installs kubeadm on the machine.
"""
logger = XprLogger()
if not linux_utils.check_root():
logger.fatal("Please run this as root")
logger.info("Installing Kubeadm...")
try:
swapoff = 'swapoff -a'
self.executor.execute(swapoff) # turns swap off
add_key = 'curl -s ' \
'https://packages.cloud.google.com/apt/doc/apt-key.gpg ' \
'| apt-key add -'
self.executor.execute(add_key)
add_list_path = '/etc/apt/sources.list.d/kubernetes.list'
add_list_content = 'deb https://apt.kubernetes.io/ ' \
'kubernetes-xenial main'
linux_utils.write_to_file(add_list_content, add_list_path, "a")
install_kubeadm = 'apt-get update && apt-get install ' \
'-y kubelet kubeadm kubectl'
self.executor.execute(install_kubeadm) # installs kubeadm
hold_kubeadm = 'apt-mark hold kubelet kubeadm kubectl'
self.executor.execute(hold_kubeadm)
except CommandExecutionFailedException as e:
logger.error("Failed to install Kubeadm. \n{}".format(str(e)))
return False
return True
def execute(self, parameters=None):
"""
installs worker node on the machine.
"""
logger = XprLogger()
if not linux_utils.check_root():
logger.fatal("Please run this as root")
self.cleanup()
logger.info("Initialising Kubernetes worker node...")
try:
if parameters and self.PARAMETER_MASTER_IP in parameters:
master_ip = parameters[self.PARAMETER_MASTER_IP]
else:
master_ip = input("Enter the IP address of the master"
" node you want to join:")
path = '/mnt/nfs/data/k8/k8_clusters/{}/{}.txt'. \
format(master_ip, master_ip)
with open(path, "r") as f:
join_command = f.read() # extract join command
self.executor.execute(
join_command) # run command to join the cluster
except CommandExecutionFailedException as e:
logger.error("Failed to setup worker node. \n{}".format(str(e)))
return False
return True
def install(self, **kwargs):
"""
- Sets up the root password
- Installs xprctl
"""
if not check_root():
raise PermissionError("Need sudo permission")
# Setup the root password
root_user = self.config["vms"]["username"]
root_pass = self.config["vms"]["password"]
try:
self.set_password(root_user, root_pass)
except RuntimeError:
raise RuntimeError("Password Setup Failed")
# Updating sshd service to allow remote root access
with open("/etc/ssh/sshd_config", "a") as ssh_conf:
ssh_conf.write("\nPermitRootLogin yes\n")
self.execute_command("service sshd restart")
# Assumption, it is being run from the root folder
self.execute_command("pip3 install .")
return True
def pull_image(self):
if not check_root():
print("Please run the program using sudo privileges")
raise PermissionDeniedException("Permission Denied. Run with admin rights.")
try:
self.client.images.pull(self.config[self.KIBANA_SECTION][self.KIBANA_IMAGE])
except docker.errors.APIError as err:
print("The server returns an error. \n{}".format(str(err)))
raise err
def pull_image(self):
if not check_root():
self.logger.error("Please run the program using sudo privileges")
raise PermissionDeniedException(
"Permission Denied. Run with admin rights.")
try:
self.client.images.pull(
self.config[self.LDAP_SECTION][self.LDAP_IMAGE])
self.logger.info("Successfully pulled LDAP docker image.")
except docker.errors.APIError as err:
self.logger.error("The server returns an error. \n{}".format(
str(err)))
raise err
def setup_logstash(self):
if not check_root():
print("Please run the program using sudo privileges")
raise PermissionDeniedException(
"Permission Denied. Run with admin rights.")
if not os.path.exists(
self.config[self.LOGSTASH_SECTION][self.LOGSTASH_CONF]):
print(
"Unable to find the logstash config folder at mentioned path")
raise FileNotFoundError
try:
self.pull_image()
self.update_logstash_config()
self.client.containers.run(
self.config[self.LOGSTASH_SECTION][self.LOGSTASH_IMAGE],
volumes={
self.config[self.LOGSTASH_SECTION][self.LOGSTASH_CONF]: {
'bind': '/usr/share/logstash/config/',
'mode': 'rw'
}
},
name=self.config[self.LOGSTASH_SECTION][
self.LOGSTASH_CONTAINER],
ports={
"5000":
self.config[self.LOGSTASH_SECTION][self.LOGSTASH_PORT],
"9600": "9300"
},
detach=True,
restart_policy={
"Name": "on-failure",
"MaximumRetryCount": 5
})
except docker.errors.ContainerError as err:
print("The container exits with a non-zero exit code. \n{}".format(
str(err)))
raise err
except docker.errors.ImageNotFound as err:
print("The specified image does not exist. \n{}".format(str(err)))
raise err
except docker.errors.APIError as err:
print("The server returns an error. \n{}".format(str(err)))
raise err
except KeyError as err:
print("Key not present. \n{}".format(str(err)))
raise err
return
def execute(self, **kwargs):
"""
Mounts the NFS on the VM
"""
logger = XprLogger()
if not linux_utils.check_root():
logger.fatal("Please run this as root")
logger.info("Mounting NFS File")
subnet_to_nfs_map = self.config[self.NFS_SECTION][self.SUBNET_MAP_KEY]
ip_address = linux_utils.get_ip_address()
matched_nfs = None
for nfs, subnet in subnet_to_nfs_map.items():
logger.info("Matching {} {}".format(subnet, ip_address))
check = re.match(subnet, ip_address)
print(check)
if check:
matched_nfs = nfs
break
if not matched_nfs:
logger.info("Could not determine nfs value")
return False
mount_location = self.config[self.NFS_SECTION][self.MOUNT_LOCATION_KEY]
nfs_location = self.config[self.NFS_SECTION][self.NFS_LOCATION_KEY]
mount_script = "mount {}:{} {}".format(matched_nfs, nfs_location,
mount_location)
logger.info("Mounting {}".format(mount_script))
try:
linux_utils.create_directory(mount_location, 0o755)
self.executor.execute(mount_script)
logger.info("Mount Succesful")
logger.info("Updating fstab file")
with open(self.FSTAB_FILE, "a+") as f:
fstab_statement = "{}:{} {} nfs " \
"auto,nofail,noatime,nolock,intr,tcp," \
"actimeo=1800 0 0" \
.format(matched_nfs, nfs_location, mount_location)
logger.info(
"Updating fstab file with {}".format(fstab_statement))
f.write(fstab_statement)
logger.info("Update Successful")
except CommandExecutionFailedException as e:
logger.error("Script Failed to run = {}\n{}".format(
mount_script, str(e)))
return False
return True
def setup_ldap(self):
if not check_root():
self.logger.error("Please run the program using sudo privileges")
raise PermissionDeniedException(
"Permission Denied. Run with admin rights.")
try:
self.pull_image()
self.client.containers.run(
self.config[self.LDAP_SECTION][self.LDAP_IMAGE],
name=self.config[self.LDAP_SECTION][self.LDAP_CONTAINER],
environment={
"LDAP_ORGANISATION": "abzooba",
"LDAP_DOMAIN": "abzooba.com",
"LDAP_ADMIN_PASSWORD": "******"
},
ports={
"389": "389",
"636": "636"
},
detach=True,
restart_policy={
"Name": "on-failure",
"MaximumRetryCount": 5
})
self.logger.info("Ldap docker service successfully started")
except docker.errors.ContainerError as err:
self.logger.error(
"The container exits with a non-zero exit code. \n{}".format(
str(err)))
raise err
except docker.errors.ImageNotFound as err:
self.logger.error(
"The specified image does not exist. \n{}".format(str(err)))
raise err
except docker.errors.APIError as err:
self.logger.error("The server returns an error. \n{}".format(
str(err)))
raise err
except KeyError as err:
self.logger.error("Key not present. \n{}".format(str(err)))
raise err
return
def setup_elastic_search(self):
if not check_root():
print("Please run the program using sudo privileges")
raise PermissionDeniedException(
"Permission Denied. Run with admin rights.")
try:
self.pull_image()
self.client.containers.run(
self.config[self.ELASTIC_SECTION][self.ELASTIC_IMAGE],
volumes={
self.config[self.ELASTIC_SECTION][self.ELASTIC_SEARCH_DUMP_NAME]:
{
'bind': '/usr/share/elasticsearch/data/',
'mode': 'rw'
}
},
environment={"discovery.type": "single-node"},
ports={
"9200":
self.config[self.ELASTIC_SECTION][self.ELASTIC_PORT]
},
name=self.config[self.ELASTIC_SECTION][self.ELASTIC_CONTAINER],
detach=True,
restart_policy={
"Name": "on-failure",
"MaximumRetryCount": 5
})
except docker.errors.ContainerError as err:
print("The container exits with a non-zero exit code. \n{}".format(
str(err)))
raise err
except docker.errors.ImageNotFound as err:
print("The specified image does not exist. \n{}".format(str(err)))
raise err
except docker.errors.APIError as err:
print("The server returns an error. \n{}".format(str(err)))
raise err
except KeyError as err:
print("Key not present. \n{}".format(str(err)))
raise err
return
def execute(self):
"""
installs kubernetes dashboard on the machine.
"""
logger = XprLogger()
if not linux_utils.check_root():
logger.fatal("Please run this as root")
logger.info("Setting up the Kubernetes dashboard...")
try:
deploy_dashboard = 'kubectl create -f https://raw.githubusercontent' \
'.com/kubernetes/dashboard/master/aio/deploy' \
'/recommended/kubernetes-dashboard.yaml'
self.executor.execute(deploy_dashboard) # creates deployment
nodeport = """kubectl -n kube-system patch service \
kubernetes-dashboard --type='json' -p \
'[{"op":"replace","path":"/spec/type","value":"NodePort"}]'"""
self.executor.execute(nodeport) # exposes dashboard
constant_port = """kubectl -n kube-system patch service \
kubernetes-dashboard --type='json' -p \
'[{"op":"replace","path":"/spec/ports/0/nodePort","value":30252}]'"""
self.executor.execute(constant_port) # sets constant port
content_path = '/opt/xpresso.ai/config/kubernetes-dashboard-access.yaml'
with open(content_path, "r") as f:
content = f.read()
path = '/etc/kubernetes/kube-dashboard-access.yaml'
linux_utils.write_to_file(content, path, "w+")
dashboard_access = 'kubectl create -f {}'.format(path)
self.executor.execute(dashboard_access) # grants permission
skip_login = """kubectl patch deployment -n kube-system \
kubernetes-dashboard --type='json' -p='[{"op": "add", "path": \
"/spec/template/spec/containers/0/args/1", \
"value":"--enable-skip-login" }]'"""
self.executor.execute(skip_login) # enables skip login
except CommandExecutionFailedException as e:
logger.error("Failed to setup dashboard. \n{}".format(str(e)))
return False
return True
def setup_kibana(self):
if not check_root():
print("Please run the program using sudo privileges")
raise PermissionDeniedException("Permission Denied. Run with admin rights.")
try:
self.pull_image()
self.client.containers.run(self.config[self.KIBANA_SECTION][self.KIBANA_IMAGE],
environment={"ELASTICSEARCH_URL": self.config[self.KIBANA_SECTION][self.ELASTIC_SEARCH_IP]},
ports={"5601":self.config[self.KIBANA_SECTION][self.KIBANA_PORT]}, name= self.config[self.KIBANA_SECTION][self.KIBANA_CONTAINER],
detach=True, restart_policy={"Name": "on-failure", "MaximumRetryCount": 5})
except docker.errors.ContainerError as err:
print("The container exits with a non-zero exit code. \n{}".format(str(err)))
raise err
except docker.errors.ImageNotFound as err:
print("The specified image does not exist. \n{}".format(str(err)))
raise err
except docker.errors.APIError as err:
print("The server returns an error. \n{}".format(str(err)))
raise err
except KeyError as err:
print("Key not present. \n{}".format(str(err)))
raise err
return
def execute(self):
"""
installs kubernetes master node on the machine.
"""
logger = XprLogger()
if not linux_utils.check_root():
logger.fatal("Please run this as root")
logger.info("Initialising Kubernetes master node...")
try:
pod_network_cidr = self.config[self.PACKAGES][self.KUBE_SECTION][
self.CIDR_KEY]
init = 'kubeadm init --token-ttl=0 --pod-network-cidr={}'.format(
pod_network_cidr)
(_, output, _) = self.executor.execute_with_output(init)
output = output.splitlines()
join_command = (output[-2].decode("utf-8").rstrip('\\') +
output[-1].decode("utf-8"))
# waiting time for master node to become active
time.sleep(90)
master_ip = linux_utils.get_ip_address()
cluster_path = '/mnt/nfs/data/k8/k8_clusters/' \
'{}'.format(master_ip)
linux_utils.create_directory(cluster_path, 0o755)
join_filename = '{}/{}.txt'.format(cluster_path, master_ip)
linux_utils.write_to_file(join_command, join_filename, "w+")
if not os.path.isfile(join_filename):
logger.error('Failed to write join command to file. Exiting.')
raise CommandExecutionFailedException
kubeconfig = 'KUBECONFIG=/etc/kubernetes/admin.conf'
environment_path = '/etc/environment'
linux_utils.write_to_file(kubeconfig, environment_path, "a")
os.environ["KUBECONFIG"] = "/etc/kubernetes/admin.conf"
kube_directory = '$HOME/.kube'
linux_utils.create_directory(kube_directory, 0o755)
copy_config = 'sudo cp -f /etc/kubernetes/admin.conf' \
' $HOME/.kube/config'
self.executor.execute(copy_config)
chown = 'sudo chown $(id -u):$(id -g) $HOME/.kube/config'
self.executor.execute(chown)
flannel = 'kubectl apply -f https://raw.githubusercontent.com' \
'/coreos/flannel/master/Documentation/kube-flannel.yml'
self.executor.execute(flannel)
generate_api_token = "kubectl get secret $(kubectl get " \
"serviceaccount default -o jsonpath=" \
"'{.secrets[0].name}') -o jsonpath=" \
"'{.data.token}' | base64 --decode"
status, stdout, stderr = self.executor.execute_with_output(
generate_api_token)
if status != 0 or len(stderr.decode('utf-8')):
raise CommandExecutionFailedException(
"Token generation failed")
token = stdout.decode("utf-8")
self.persistence_manager.update("nodes", {"address": master_ip},
{"token": token})
api_access = 'kubectl create clusterrolebinding permissive-binding \
--clusterrole=cluster-admin \
--user=admin \
--user=kubelet \
--group=system:serviceaccounts'
self.executor.execute(api_access)
docker_secret = \
'kubectl create secret docker-registry dockerkey ' \
'--docker-server https://dockerregistry.xpresso.ai/ ' \
'--docker-username xprdocker --docker-password Abz00ba@123'
self.executor.execute(docker_secret)
except CommandExecutionFailedException as e:
logger.error("Failed to initialise master. \n{}".format(str(e)))
return False
return True