def complete_installation():
print(os.linesep)
msg = "This Kubernetes environment"
warnings = Log.get_warning_count()
errors = Log.get_error_count()
if errors > 0 and warnings > 0:
msg = "{0} had {1} error(s) and {2} warning(s) during the bootstraping process for MapR".format(
msg, errors, warnings)
Log.error(msg)
elif errors > 0 and warnings == 0:
msg = "{0} had {1} error(s) during the bootstraping process for MapR".format(
msg, errors)
Log.error(msg)
elif errors == 0 and warnings > 0:
msg = "{0} had {1} warnings(s) during the bootstraping process for MapR".format(
msg, warnings)
Log.warning(msg)
else:
msg = "{0} has been successfully bootstrapped for MapR".format(msg)
Log.info(msg, True)
Log.info(
"MapR components can now be created via the newly installed operators",
True)
if errors > 0 or warnings > 0:
msg = "Please check the bootstrap log file for this session here: {0}".format(
Log.get_log_filename())
Log.warning(msg)
Log.info("")
def invoke_gcloud(cmd):
response, status = OSCommand.run2("gcloud {0}".format(cmd))
if status != 0:
Log.error("Could not create GKE Cluster: {0}: {1}".format(
status, response))
BootstrapBase.exit_application(101)
return response, status
def complete_uninstallation():
print(os.linesep)
msg = "This Kubernetes environment"
warnings = Log.get_warning_count()
errors = Log.get_error_count()
if errors > 0 and warnings > 0:
msg = "{0} had {1} error(s) and {2} warning(s) during the uninstall process for MapR".format(
msg, errors, warnings)
Log.error(msg)
elif errors > 0 and warnings == 0:
msg = "{0} had {1} error(s) during the uninstall process for MapR".format(
msg, errors)
Log.error(msg)
elif errors == 0 and warnings > 0:
msg = "{0} had {1} warnings(s) during the uninstall process for MapR".format(
msg, warnings)
Log.warning(msg)
else:
msg = "{0} has had MapR successfully uninstalled".format(msg)
Log.info(msg, True)
if errors > 0 or warnings > 0:
msg = "Please check the bootstrap log file for this session here: {0}".format(
Log.get_log_filename())
Log.warning(msg)
Log.info("")
def build_cloud(self):
self.cluster_name = self.prompts.prompt("Enter cluster name",
self.cluster_name,
key_name="GKE_CLUSTER_NAME")
self.nodes = self.prompts.prompt_integer("Enter number of nodes",
self.nodes,
1,
key_name="GKE_NODE_COUNT")
self.disks = self.prompts.prompt_integer(
"Enter number of local SSD disks for MapR FS. Each disk will be a fixed 375GB",
self.disks,
1,
key_name="GKE_NUM_DISKS")
self.instance_type = self.prompts.prompt("GKE compute instance type?",
self.instance_type,
key_name="GKE_INSTANCE_TYPE")
if self.alpha:
self.k8s_version = self.prompts.prompt("Kubernetes version?",
self.k8s_alpha_version,
key_name="GKE_K8S_VERSION")
else:
self.k8s_version = self.prompts.prompt("Kubernetes version?",
self.k8s_version,
key_name="GKE_K8S_VERSION")
self.zone = self.prompts.prompt("GCE Zone to deploy into?",
self.zone,
key_name="GKE_ZONE")
self.project = self.prompts.prompt("GCE project id?",
self.project,
key_name="GKE_PROJECT_ID")
self.user = self.prompts.prompt("GCE user id?",
self.user,
key_name="GKE_USER")
# self.image_type = self.prompts.prompt("GKE image type?", self.image_type)
Log.info("Using GKE compute image type: {0}".format(self.image_type),
True)
if self.alpha:
Log.info("Using alpha Kubernetes version", True)
else:
Log.info("Using non-alpha Kubernetes version", True)
if not self.prompts.prompt_boolean(
"Ready to create Google GKS cluster. Do you want to continue?",
True,
key_name="GKE_CREATE"):
Log.error("Exiiting since user is not ready to continue")
BootstrapBase.exit_application(100)
before = time.time()
self.create_k8s_cluster()
after = time.time()
diff = int(after - before)
Log.info(
"Cluster creation took {0}m {1}s".format(diff / 60, diff % 60),
True)
def configure_cloud(self):
cmd = GoogleCloud.CMD_ROLE_BINDING.format(self.user)
Log.info("Now we will configure RBAC for your kubernetes env...", True)
Log.info(
"Binding cluster-admin role to GCE user: {0}...".format(self.user),
True)
response, status = OSCommand.run2(cmd)
if status != 0:
Log.error("Could not bind cluster-admin role: {0}:{1}", status,
response)
Log.info("Configured GKE permissions", True)
def collect(self):
Log.debug('Checking kubectl is installed correctly...')
response, status = OSCommand.run2("command -v kubectl")
if status == 0:
Log.info("Looking good... Found kubectl")
self.operation = Validator.OPERATION_OK
else:
self.operation = Validator.OPERATION_INSTALL
Log.error(
"You will need to have kubectl installed on this machine.")
Log.error(
"To install kubectl please see: https://kubernetes.io/docs/tasks/tools/install-kubectl/"
)
def configure_kubernetes(self):
print(os.linesep)
Log.info("Ensuring proper kubernetes configuration...", True)
Log.info("Checking kubectl can connect to your kubernetes cluster...",
True)
response, status = OSCommand.run2("kubectl get nodes")
if status != 0:
Log.error(
"Cannot connect to Kubernetes. Make sure kubectl is pre-configured to communicate with a Kubernetes cluster."
)
BootstrapBase.exit_application(4)
Log.info("Looking good... Connected to Kubernetes", True)
if self.cloud_instance is not None:
self.cloud_instance.configure_cloud()
def collect(self):
Log.debug('Checking for oc (OpenShift CLI) installation...')
response, status = OSCommand.run2("command -v oc")
if status == 0:
Log.info("Looking good... Found oc (OpenShift CLI) installed",
True)
self.operation = Validator.OPERATION_OK
else:
self.operation = Validator.OPERATION_NONE
Log.error(
"You will need to have oc (OpenShift CLI) installed on this machine."
)
Log.error(
"To install oc please see: https://docs.openshift.com/container-platform/3.11/cli_reference/get_started_cli.html"
)
def _get_json(self):
Log.info("Retrieving node information...", stdout=True)
result, status = self.k8s.run_get("nodes -o=json")
if status != 0:
return None
self._json = json.loads(result)
if self._json is None:
Log.error("No JSON was returned from get nodes command")
return
self._items = self._json.get("items")
if self._items is None:
Log.error("No items dictonary in get nodes JSON")
return
self._node_count = len(self._items)
def validate_nodes(self):
print(os.linesep)
Log.info(
"We must validate and annotate your Kubernetes nodes. "
"MapR node validation pods will be installed.",
stdout=True)
agree = self._prompts.prompt_boolean("Do you agree?",
True,
key_name="AGREEMENT_VALIDATE")
if not agree:
Log.error("Exiting due to non-agreement...")
BootstrapBase.exit_application(2)
# TODO: Add node exclusion code here
# exclude = self._prompts.prompt_boolean("Do you want to exclude any nodes?", False, key_name="EXCLUDE_NODES")
# if exclude:
# Log.error("Operation not currently supported...")
# BootstrapBase.exit_application(6)
print("")
def collect(self):
Log.debug("Getting Python information...")
python_major, python_version = self._get_python_version()
Log.info("Python version: {0}".format(python_version))
self.results[Validator.FOUND] = True
self.results[Validator.VERSION] = python_version
if python_major == 2:
pmin = PythonValidator.PYTHON2_MIN
pmax = PythonValidator.PYTHON2_MAX
elif python_major == 3:
pmin = PythonValidator.PYTHON3_MIN
pmax = PythonValidator.PYTHON3_MAX
if python_version <= PythonValidator.PYTHON3_ERROR_MAX:
Log.error("The virtual environments created with your python version {0} are incompatible. "
"Please use Python 3.3 or greater".format(python_version))
self.operation = Validator.OPERATION_INSTALL
return
else:
Log.error("The major Python version '{0}' is not supported; Only version 2 and 3 supported".format(python_major))
self.operation = Validator.OPERATION_TOO_NEW
return
expected = "Expected versions between {0} and {1} or between {2} and {3}"\
.format(PythonValidator.PYTHON2_MIN, PythonValidator.PYTHON2_MAX,
PythonValidator.PYTHON3_MIN, PythonValidator.PYTHON3_MAX)
if python_version > pmax or python_version < pmin:
Log.warning("The Python version on this system is {0}. {1}"
.format(python_version, expected))
self.operation = Validator.OPERATION_WARNING
else:
Log.debug("The Python version on this system is compatible")
self.operation = Validator.OPERATION_OK
def create_k8s_cluster(self):
Log.info(
"Creating cluster with {0} nodes of type {1} with {2} local ssd(s) of size 375GB..."
.format(self.nodes, self.instance_type, self.disks), True)
args = "--zone {0} ".format(self.zone)
args += "--username admin "
args += "--cluster-version {0} ".format(self.k8s_version)
args += "--machine-type {0} ".format(self.instance_type)
args += "--image-type {0} ".format(self.image_type)
args += "--disk-size {0} ".format(self.disk_size_on_node)
args += "--disk-type {0} ".format(self.disk_type_on_node)
args += "--num-nodes {0} ".format(self.nodes)
args += "--network default "
args += "--enable-cloud-logging "
args += "--enable-cloud-monitoring "
args += "--subnetwork default "
args += "--scopes https://www.googleapis.com/auth/compute,https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append "
if self.alpha:
args += "--enable-kubernetes-alpha "
args += "--no-enable-autorepair "
self.invoke_alpha_cluster(args)
else:
self.invoke_stable_cluster(args)
Log.info("Node log follows after cluster creation. One moment...",
True)
cmd = "container clusters get-credentials {0} --zone {1} --project {2}".format(
self.cluster_name, self.zone, self.project)
result, status = self.invoke_gcloud(cmd)
Log.info(result, True)
# TODO Got to be a better way to get this filtered list
result, status = OSCommand.run2(
"kubectl get nodes | grep Ready | cut -d' ' -f1")
if status != 0:
Log.error("Could not get list of nodes: {0}: {1}".format(
status, result))
BootstrapBase.exit_application(103)
nodes = result.split("\n")
if len(nodes[-1]) == 0:
nodes = nodes[:-1]
Log.info(
"After cluster creation, {0} node(s) were found".format(
len(nodes)), True)
all_threads = list()
for node in nodes:
t = Thread(target=self.create_disks_and_attach,
args=(node, ),
name=node)
all_threads.append(t)
t.start()
time.sleep(0.05)
for thread in all_threads:
thread.join(timeout=10 * 60)
if thread.is_alive():
Log.error(
"Thread for node {0} did not complete in the supplied time limit"
.format(thread.name))