def test_all_hosts_config(self):
SUMMARY = """
{
"items" : [ {
"name" : "blacklisted_parcel_products",
"value" : "foo,bar"
} ]
}
"""
FULL = """
{
"items" : [ {
"name" : "blacklisted_parcel_products",
"value" : "foo,bar",
"required" : false,
"default" : "",
"displayName" : "Blacklisted Products",
"description" : "Parcels for blacklisted products will not be distributed to the host, nor activated for process execution. Already distributed parcels will be undistributed. Already running process will not be affected until the next restart.",
"validationState" : "OK"
}, {
"name" : "rm_enabled",
"required" : false,
"default" : "false",
"displayName" : "Enable Resource Management",
"description" : "Enables resource management for all roles on this host.",
"validationState" : "OK",
"validationWarningsSuppressed" : false
} ]
}
"""
resource = utils.MockResource(self)
cms = ClouderaManager(resource)
resource.expect("GET",
"/cm/allHosts/config",
retdata=json.loads(SUMMARY))
cfg = cms.get_all_hosts_config()
self.assertIsInstance(cfg, dict)
self.assertEqual(1, len(cfg))
self.assertEqual('foo,bar', cfg.get('blacklisted_parcel_products'))
resource.expect("GET",
"/cm/allHosts/config",
params={'view': 'full'},
retdata=json.loads(FULL))
cfg = cms.get_all_hosts_config(view='full')
self.assertIsInstance(cfg, dict)
self.assertEqual(2, len(cfg))
self.assertIsInstance(cfg['blacklisted_parcel_products'], ApiConfig)
self.assertFalse(cfg['blacklisted_parcel_products'].required)
self.assertEqual('OK', cfg['rm_enabled'].validationState)
cfg = {'blacklisted_parcel_products': 'bar'}
resource.expect("PUT",
"/cm/allHosts/config",
data=config_to_json(cfg),
retdata=json.loads(SUMMARY))
cms.update_all_hosts_config(cfg)
def test_all_hosts_config(self):
SUMMARY = """
{
"items" : [ {
"name" : "blacklisted_parcel_products",
"value" : "foo,bar"
} ]
}
"""
FULL = """
{
"items" : [ {
"name" : "blacklisted_parcel_products",
"value" : "foo,bar",
"required" : false,
"default" : "",
"displayName" : "Blacklisted Products",
"description" : "Parcels for blacklisted products will not be distributed to the host, nor activated for process execution. Already distributed parcels will be undistributed. Already running process will not be affected until the next restart.",
"validationState" : "OK"
}, {
"name" : "rm_enabled",
"required" : false,
"default" : "false",
"displayName" : "Enable Resource Management",
"description" : "Enables resource management for all roles on this host.",
"validationState" : "OK",
"validationWarningsSuppressed" : false
} ]
}
"""
resource = utils.MockResource(self)
cms = ClouderaManager(resource)
resource.expect("GET", "/cm/allHosts/config", retdata=json.loads(SUMMARY))
cfg = cms.get_all_hosts_config()
self.assertIsInstance(cfg, dict)
self.assertEqual(1, len(cfg))
self.assertEqual('foo,bar', cfg.get('blacklisted_parcel_products'))
resource.expect("GET", "/cm/allHosts/config", params={ 'view' : 'full' },
retdata=json.loads(FULL))
cfg = cms.get_all_hosts_config(view='full')
self.assertIsInstance(cfg, dict)
self.assertEqual(2, len(cfg))
self.assertIsInstance(cfg['blacklisted_parcel_products'], ApiConfig)
self.assertFalse(cfg['blacklisted_parcel_products'].required)
self.assertEqual('OK', cfg['rm_enabled'].validationState)
cfg = { 'blacklisted_parcel_products' : 'bar' }
resource.expect("PUT", "/cm/allHosts/config", data=config_to_json(cfg),
retdata=json.loads(SUMMARY))
cms.update_all_hosts_config(cfg)
def set_up_cluster(cm_host, host_list):
print "Setting up CDH cluster..."
api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=7)
cm = ClouderaManager(api)
print "Creating mgmg service."
try:
service_setup = ApiServiceSetupInfo(name=cm_service_name, type="MGMT")
cm.create_mgmt_service(service_setup)
except ApiException as exc:
if exc.code != 400:
print "create MGMT service failed: " + exc
exit(1)
print "Installing hosts. This might take a while."
cmd = cm.host_install(host_username, host_list,
password=host_password).wait()
if cmd.success != True:
print "cm_host_install failed: " + cmd.resultMessage
exit(2)
print "Auto-assign roles and auto-configure the CM service"
if not is_cluster_installed(api):
cm.auto_assign_roles()
cm.auto_configure()
print "Creating cluster."
if not is_cluster_installed(api):
cluster = create_cluster(api, cluster_name, cdh_version)
cluster.add_hosts(host_list)
cluster = api.get_cluster(cluster_name)
cdh_parcel = get_cdh_parcel(cluster)
print "Downloading CDH parcel. This might take a while."
if cdh_parcel.stage == "AVAILABLE_REMOTELY":
cdh_parcel = wait_for_parcel(cdh_parcel.start_download(), api,
cdh_parcel, cluster_name, 'DOWNLOADED')
print "Distributing CDH parcel. This might take a while."
if cdh_parcel.stage == "DOWNLOADED":
cdh_parcel = wait_for_parcel(cdh_parcel.start_distribution(), api,
cdh_parcel, cluster_name, 'DISTRIBUTED')
print "Activating CDH parcel. This might take a while."
if cdh_parcel.stage == "DISTRIBUTED":
cdh_parcel = wait_for_parcel(cdh_parcel.activate(), api, cdh_parcel,
cluster_name, 'ACTIVATED')
# if cdh_parcel.stage != "ACTIVATED":
# print "CDH parcel activation failed. Parcel in stage: " + cdh_parcel.stage
# exit(14)
print "Inspecting hosts. This might take a few minutes."
cmd = cm.inspect_hosts()
while cmd.success == None:
cmd = cmd.fetch()
if cmd.success != True:
print "Host inpsection failed!"
exit(8)
print "Hosts successfully inspected: \n" + cmd.resultMessage
print "Creating specified services."
for s in service_types_and_names.keys():
try:
cluster.get_service(service_types_and_names[s])
except:
print "Creating service: " + service_types_and_names[s]
service = cluster.create_service(service_types_and_names[s], s)
slaves = [host for host in host_list if 'slave' in host]
edges = [host for host in host_list if 'edge' in host]
#assign master roles to master node
for service in cluster.get_all_services():
if service.name == 'HDFS-1':
service.create_role('NAMENODE-1', 'NAMENODE', cm_host)
service.create_role('SECONDARYNAMENODE', 'SECONDARYNAMENODE',
cm_host)
service.create_role('BALANCER-1', 'BALANCER', cm_host)
service.create_role('HTTPFS-1', 'HTTPFS', cm_host)
service.create_role('HDFS-GW_MASTER1', 'GATEWAY', cm_host)
for (i, edge) in enumerate(edges):
service.create_role('HDFS-GW_EDGE%s' % i, 'GATEWAY', edge)
for (i, slave) in enumerate(slaves):
service.create_role('DATANODE-%s' % i, 'DATANODE', slave)
if service.name == 'ZOOKEEPER-1':
service.create_role('ZOOKEEPERSERVER-1', 'SERVER', cm_host)
if service.name == 'HBASE-1':
service.create_role('MASTER-1', 'MASTER', cm_host)
service.create_role('HBASETHRIFTSERVER-1', 'HBASETHRIFTSERVER',
cm_host)
for (i, slave) in enumerate(slaves):
service.create_role('HBASE-RS-%s' % i, 'REGIONSERVER', slave)
if service.name == 'HUE-1':
service.create_role('HUE-MASTER1', 'HUE_SERVER', cm_host)
service.create_role('HUE-LB_MASTER1', 'HUE_LOAD_BALANCER', cm_host)
for (i, edge) in enumerate(edges):
service.create_role('HUE-EDGE%s' % i, 'HUE_SERVER', edge)
if service.name == 'HIVE-1':
service.create_role('HIVEMETASTORE-1', 'HIVEMETASTORE', cm_host)
service.create_role('HIVESERVER-1', 'HIVESERVER2', cm_host)
service.create_role('HIVE-GW_MASTER1', 'GATEWAY', cm_host)
for (i, edge) in enumerate(edges):
service.create_role('HIVE-GW_EDGE%s' % i, 'GATEWAY', edge)
for (i, slave) in enumerate(slaves):
service.create_role('HIVE-GW_SLAVE%s' % i, 'GATEWAY', slave)
if service.name == 'IMPALA-1':
service.create_role('STATESTORE-1', 'STATESTORE', cm_host)
service.create_role('CATALOGSERVER-1', 'CATALOGSERVER', cm_host)
for (i, slave) in enumerate(slaves):
service.create_role('IMPALAD-%s' % i, 'IMPALAD', slave)
if service.name == 'OOZIE-1':
service.create_role('OOZIE_SERVER-1', 'OOZIE_SERVER', cm_host)
if service.name == 'SPARK_ON_YARN-1':
service.create_role('SPARK_YARN_HISTORY_SERVER-1',
'SPARK_YARN_HISTORY_SERVER', cm_host)
service.create_role('SPARK_ON_YARN-GW_MASTER%s' % i, 'GATEWAY',
cm_host)
for (i, edge) in enumerate(edges):
service.create_role('SPARK_ON_YARN-GW_EDGE%s' % i, 'GATEWAY',
edge)
for (i, slave) in enumerate(slaves):
service.create_role('SPARK_ON_YARN-GW_SLAVE%s' % i, 'GATEWAY',
slave)
if service.name == 'SQOOP-1':
service.create_role('SQOOP_SERVER-1', 'SQOOP_SERVER', cm_host)
if service.name == 'YARN-1':
service.create_role('RESOURCEMANAGER-1', 'RESOURCEMANAGER',
cm_host)
service.create_role('JOBHISTORY-1', 'JOBHISTORY', cm_host)
for (i, slave) in enumerate(slaves):
service.create_role('NODEMANAGER-%s' % i, 'NODEMANAGER', slave)
#print "Auto assigning roles."
#cluster.auto_assign_roles()
cluster.auto_configure()
print "Updating Hive config."
hive_metastore_host = cm_host # let's assume that
hive = cluster.get_service(service_types_and_names["HIVE"])
hive_config = { "hive_metastore_database_host" : hive_metastore_host, \
"hive_metastore_database_name" : hive_metastore_name, \
"hive_metastore_database_user" : hive_metastore_user, \
"hive_metastore_database_password" : hive_metastore_password, \
"hive_metastore_database_port" : hive_metastore_database_port, \
"hive_metastore_database_type" : hive_metastore_database_type }
hive.update_config(hive_config)
print "Updating Hue config."
hue_db_host = cm_host # let's assume that
hue = cluster.get_service(service_types_and_names["HUE"])
hue_config = { "database_host" : hue_db_host, \
"database_name" : hue_db_name, \
"database_user" : hue_db_user, \
"database_password" : hue_db_password, \
"database_port" : hue_db_port, \
"database_type" : hue_db_type }
hue.update_config(hue_config)
# Set Java version to OpenJDK
cm.update_all_hosts_config({'java_home': '/usr/lib/jvm/java-openjdk'})
print "Starting management service."
cm_service = cm.get_service()
cm_service.start().wait()
print "Excuting first run command. This might take a while."
cmd = cluster.first_run().wait()
if cmd.success != True:
print "The first run command failed: " + cmd.resultMessage
exit(11)
print "First run successfully executed. Your cluster has been set up!"
config = cm.get_config(view='full')
repolist = config['REMOTE_PARCEL_REPO_URLS']
value = repolist.value or repolist.default
value += ',' + anaconda_repo
cm.update_config({'REMOTE_PARCEL_REPO_URLS': value})
sleep(10)
cluster = api.get_cluster(cluster_name)
parcel = cluster.get_parcel('Anaconda', anaconda_parcel_version)
print "Downloading Anaconda parcel. This might take a while."
if parcel.stage == "AVAILABLE_REMOTELY":
parcel = wait_for_parcel(parcel.start_download(), api, parcel,
cluster_name, 'DOWNLOADED')
print "Distributing Anaconda parcel. This might take a while."
if parcel.stage == "DOWNLOADED":
parcel = wait_for_parcel(parcel.start_distribution(), api, parcel,
cluster_name, 'DISTRIBUTED')
print "Activating Anaconda parcel. This might take a while."
if parcel.stage == "DISTRIBUTED":
parcel = wait_for_parcel(parcel.activate(), api, parcel, cluster_name,
'ACTIVATED')
print "Anaconda is now installed."