def test_get_clusters_two_clusters(self):
with mock.patch("cluster_config.log.fatal") as log, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service,\
mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters:
log.return_value = MagicMock()
get_service.return_value = MagicMock()
api_source.return_value = MagicMock()
all_clusters.return_value = [
type('obj', (object, ), {
'name': "123",
"displayName": "123"
}),
type('obj', (object, ), {
'name': "1234",
"displayName": "1234"
})
]
cluster = Cluster("host", "Port", "username", "password", "123")
assert cluster.cluster.name == "123"
cluster = Cluster("host", "Port", "username", "password", "1234")
assert cluster.cluster.name == "1234"
def test_set_services(self):
cc.cdh.Service = MagicMock()
with mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_services") as cdh_services,\
mock.patch("cluster_config.cdh.cluster.Cluster._service") as services:
api_source.return_value = MagicMock()
all_clusters.return_value = [
type('obj', (object, ), {'name': "123"})
]
cdh_services.return_value = [
type('obj', (object, ), {
'name': "yarn",
"key": "YARN"
})
]
services.return_value = return_value = type(
'obj', (object, ), {
'name': "yarn",
"key": "YARN"
})
#services.return_value = {}
cluster = Cluster("host", "Port", "username", "password",
"cluster_name")
assert cluster.yarn
assert cluster.services["YARN"]
def test_get_clusters_two_clusters_invalid_index(self):
with mock.patch("cluster_config.log.fatal") as logg, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service,\
mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters,\
mock.patch("__builtin__.input") as input_mock:
logg.return_value = MagicMock()
get_service.return_value = MagicMock()
api_source.return_value = MagicMock()
input_mock.return_value = 9
all_clusters.return_value = [
type('obj', (object, ), {
'name': "123",
"displayName": "first",
"version": "1"
}),
type('obj', (object, ), {
'name': "1234",
"displayName": "second",
"version": "2"
})
]
Cluster("host", "Port", "username", "password", None)
assert logg.call_count == 1
def run(args, cluster=None):
if cluster is None:
cluster = Cluster(args.host, args.port, args.username, args.password,
args.cluster)
dt = datetime.datetime.now()
generate.run(args, cluster, dt)
push.run(args, cluster, dt)
def main():
from cluster_config.cdh.cluster import Cluster
#from cluster_config import push
from cluster_config.utils.cli import parse
args = parse(cli())
cluster = Cluster(args.host, args.port, args.username, args.password,
args.cluster)
run(args, cluster)
def test_missing_password(self):
with mock.patch("cluster_config.log.fatal") as log_fatal, \
mock.patch("cluster_config.cdh.cluster.Cluster._get_cluster") as get_cluster, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service:
log_fatal.return_value = MagicMock()
get_cluster.return_value = MagicMock()
get_service.return_value = MagicMock()
Cluster("host", "Port", "username", None, "cluster_name")
assert log_fatal.call_count == 1
log_fatal.assert_called_with(
"password init parameter can't be None or empty")
def run(args, cluster=None, dt=None):
if cluster is None:
cluster = Cluster(args.host, args.port, args.username, args.password, args.cluster)
if cluster:
cluster_before = save_to_json(cluster, args.path, "before")
#Read the generated configs
cdh_config_path = file.file_path(cc.CDH_CONFIG, args.path)
log.info("Reading CDH config file: {0}".format(cdh_config_path))
cdh_configs = file.open_json_conf(cdh_config_path)
user_cdh_config_path = file.file_path(cc.USER_CDH_CONFIG, args.path)
log.info("Reading user CDH config file: {0}".format(user_cdh_config_path))
user_configs = file.open_json_conf(user_cdh_config_path)
merged_config_path = None
if user_configs:
#merge config dictionaries and resolve conflicts
log.info("conflict resolution: {0}".format(args.conflict_merge))
configs = cc.utils.dict.merge_dicts(user_configs, cdh_configs, convert_conflict_merge(args.conflict_merge))
merged_config_path = file.file_path(cc.MERGED_CDH_CONFIG, args.path)
log.info("Writting merged CDH config file: {0}".format(merged_config_path))
file.write_json_conf(configs, merged_config_path)
else:
configs = cdh_configs
#if update cdh is "yes" then we iterate and update all the specified keys
if args.update_cdh == "yes":
#iterate through services, set cdh configs and possibly restart services
cluster.update_configs(configs, False if args.restart_cdh == "no" else True)
cluster_after = save_to_json(cluster, args.path, "after")
file.snapshots(args.host, "push", args.path, dt, cluster_before, cluster_after, cdh_config_path,
user_cdh_config_path, merged_config_path)
else:
log.fatal("Couldn't connect to the CDH cluster")
def test_get_clusters_zero_clusters(self):
with mock.patch("cluster_config.log.fatal") as log, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service,\
mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters:
log.return_value = MagicMock()
get_service.return_value = MagicMock()
api_source.return_value = MagicMock()
all_clusters.return_value = []
Cluster("host", "Port", "username", "password", "cluster_name")
log.assert_called_with("No clusters to configure")
assert log.call_count == 1
def test_get_clusters_one_cluster(self):
with mock.patch("cluster_config.log.info") as log_info, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service,\
mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters:
log_info.return_value = MagicMock()
get_service.return_value = MagicMock()
api_source.return_value = MagicMock()
all_clusters.return_value = [
type('obj', (object, ), {'name': "123"})
]
cluster = Cluster("host", "Port", "username", "password",
"cluster_name")
assert cluster.cluster.name == "123"
log_info.assert_called_with("cluster selected: 123")
def run(args, cluster=None, dt=None):
if cluster is None:
cluster = Cluster(args.host, args.port, args.username, args.password,
args.cluster)
if args.formula:
cluster_config_json_path = save_to_json(cluster, args.path, "before")
vars = exec_formula(cluster, args)
cdh_config_path = save_cdh_configuration(vars, args)
file.snapshots(args.host, "generate", args.path, dt, cdh_config_path,
cluster_config_json_path, args.formula,
args.formula_args)
else:
cc.utils.log.fatal("Formula file must be specified")
def run(args, cluster=None):
'''
:param args: parsed args from argparse.ArgumentParser object
:param cluster: possible cluster object
'''
if cluster is None:
cluster = Cluster(args.host, args.port, args.username, args.password,
args.cluster)
dump = "no"
if args.dump != "yes":
dump = raw_input("dump all configs[yes or no]: ").strip()
if dump == "yes" or args.dump == "yes":
for service in cluster.services:
for role in cluster.services[service].roles:
for config_group in cluster.services[service].roles[
role].config_groups:
for config in cluster.services[service].roles[
role].config_groups[config_group].configs:
print_details(cluster, config, service, role,
config_group)
service_index = pick("cluster", "service", cluster.user_cluster_name,
cluster.services)
role_index = pick("service", "role", service_index,
cluster.services[service_index].roles)
config_group_index = pick(
"role", "config group", role_index,
cluster.services[service_index].roles[role_index].config_groups)
for config in cluster.services[service_index].roles[
role_index].config_groups[config_group_index].configs:
print_details(cluster, config, service_index, role_index,
config_group_index)
run_again(args)
def test_get_clusters_two_clusters_invalid_cluster_name(self):
with mock.patch("cluster_config.log.fatal") as logg, \
mock.patch("cluster_config.cdh.cluster.Cluster._set_services") as get_service,\
mock.patch("cluster_config.cdh.cluster.Cluster._get_api_resource") as api_source,\
mock.patch("cluster_config.cdh.cluster.Cluster._cdh_get_clusters") as all_clusters:
logg.return_value = MagicMock()
get_service.return_value = MagicMock()
api_source.return_value = MagicMock()
all_clusters.return_value = [
type('obj', (object, ), {
'name': "123",
"displayName": "123"
}),
type('obj', (object, ), {
'name': "1234",
"displayName": "1234"
})
]
cluster = Cluster("host", "Port", "username", "password", "1231")
assert logg.call_count == 1
logg.assert_called_with("Couldn't find cluster: '1231'")