def test_fatal(self):
log.logger.log = MagicMock()
sys.exit = MagicMock()
log.fatal("fatal")
log.logger.log.assert_called_with(logging.FATAL, "fatal")
sys.exit.assert_called_with(1)
def select_cluster(self, clusters):
'''
Select a cluster either by trying to match self.user_cluster_name to the cluster name or display name or by
prompting the user to select a cluster.
:return: selected cluster
'''
if self.user_cluster_name:
print("Trying to find cluster: '{0}'.".format(
self.user_cluster_name))
for c in clusters:
if c.displayName == self.user_cluster_name or c.name == self.user_cluster_name:
return c
else:
print("Please choose a cluster.")
count = 0
for c in clusters:
print("Id {0}: Cluster Name: {1:20} Version: {2}".format(
(count + 1), c.name, c.version))
count += 1
cluster_index = input("Enter the clusters Id : ")
if cluster_index <= 0 or cluster_index > count:
log.fatal(
"Invalid cluster id selected: {0}.".format(cluster_index))
else:
cluster_index -= 1
print(
"You picked cluster {0}: Cluster Name: {1:20} Version: {2}"
.format(cluster_index, clusters[cluster_index].name,
clusters[cluster_index].version))
return clusters[cluster_index]
def cdh_get_clusters(self):
'''
retrieve all the clusters managed by cdh
:return: list of cluster objects
'''
clusters = None
try:
clusters = self.cmapi_resource_root.get_all_clusters()
except URLError:
log.fatal("couldn't connect to cluster management node.")
return clusters
def exec_formula(cluster, args):
log.info("using formula: {0}".format(args.formula))
user_formula_args = None
if args.formula_args:
user_formula_args_path = file.file_path(cc.FORMULA_ARGS, args.path)
log.info("Reading formula args config file: {0}".format(
user_formula_args_path))
user_formula_args = file.open_yaml_conf(user_formula_args_path)
#execute formula global variables
vars = {
"log": log,
"kb_to_bytes": convert.kb_to_bytes,
"bytes_to_kb": convert.bytes_to_kb,
"mb_to_bytes": convert.mb_to_bytes,
"bytes_to_mb": convert.bytes_to_mb,
"gb_to_bytes": convert.gb_to_bytes,
"bytes_to_gb": convert.bytes_to_gb,
"tr_to_bytes": convert.tr_to_bytes,
"bytes_to_tr": convert.bytes_to_tr
}
local = {}
config = {}
try:
execfile(args.formula, vars, local)
constants = local["constants"](cluster, log)
for member in constants:
if hasattr(constants[member], '__call__'):
if user_formula_args and member in user_formula_args:
if constants[member](user_formula_args[member]
) != user_formula_args[member]:
log.warning(
"Formula arg value '{0}' was ignored using default '{1}'. Formula arg value must adhere to these rules {2} "
.format(
user_formula_args[member],
constants[member](user_formula_args[member]),
inspect.getsource(constants[member])))
constants[member] = constants[member](
user_formula_args[member])
else:
constants[member] = constants[member](None)
config = local["formula"](cluster, log, constants)
except IOError:
log.fatal("formula file: {0} doesn't exist".format(args.formula))
return config
def get_cluster(self):
'''
Try to select a cluster if we have more than one cluster we need to ask the user. If we have 0 through fatal.
if we have exactly one make it the default selection
'''
clusters = self.cdh_get_clusters()
#if we have more than one cluster we need to pick witch one to configure against
if len(clusters) > 1:
log.info("More than one Cluster Detected.")
return self.select_cluster(clusters)
#else pick the only cluster
elif len(clusters) == 1:
log.info("cluster selected: {0}".format(clusters[0].name))
return clusters[0]
elif len(clusters) <= 0:
log.fatal("No clusters to configure")
def run(args, cluster=None, dt=None):
if cluster is None:
cluster = Cluster(args.host, args.port, args.username, args.password, args.cluster)
if cluster:
cluster_before = save_to_json(cluster, args.path, "before")
#Read the generated configs
cdh_config_path = file.file_path(cc.CDH_CONFIG, args.path)
log.info("Reading CDH config file: {0}".format(cdh_config_path))
cdh_configs = file.open_json_conf(cdh_config_path)
user_cdh_config_path = file.file_path(cc.USER_CDH_CONFIG, args.path)
log.info("Reading user CDH config file: {0}".format(user_cdh_config_path))
user_configs = file.open_json_conf(user_cdh_config_path)
merged_config_path = None
if user_configs:
#merge config dictionaries and resolve conflicts
log.info("conflict resolution: {0}".format(args.conflict_merge))
configs = cc.utils.dict.merge_dicts(user_configs, cdh_configs, convert_conflict_merge(args.conflict_merge))
merged_config_path = file.file_path(cc.MERGED_CDH_CONFIG, args.path)
log.info("Writting merged CDH config file: {0}".format(merged_config_path))
file.write_json_conf(configs, merged_config_path)
else:
configs = cdh_configs
#if update cdh is "yes" then we iterate and update all the specified keys
if args.update_cdh == "yes":
#iterate through services, set cdh configs and possibly restart services
cluster.update_configs(configs, False if args.restart_cdh == "no" else True)
cluster_after = save_to_json(cluster, args.path, "after")
file.snapshots(args.host, "push", args.path, dt, cluster_before, cluster_after, cdh_config_path,
user_cdh_config_path, merged_config_path)
else:
log.fatal("Couldn't connect to the CDH cluster")
def get_api_resource(self, host, port, username, password):
'''
Instantiate the cm_api resource, verify the host, port, username, and password is not empty or None
:param host:
:param port:
:param username:
:param password:
'''
if not host:
log.fatal("host init parameter can't be None or empty")
if not port:
log.fatal("port init parameter can't be None or empty")
if not username:
log.fatal("username init parameter can't be None or empty")
if not password:
log.fatal("password init parameter can't be None or empty")
#initialize cloudera manager connection
log.debug("Connection to cluster {0}:{1} with user {2}:{3}".format(
host, port, username, password))
self.cdh_resource_root = ApiResource(host,
server_port=port,
username=username,
password=password)