def list_clusters(env_name=None, dep_name=None):
# Using default if available
env_name = env_name if env_name else horton.cdcred.name
dep_names = [dep_name] if dep_name else list_deployments(env_name)
clusters = []
for d in dep_names:
[clusters.append(x) for x in [cd.ClustersApi(horton.cad).list(environment=env_name, deployment=d)]]
return [x for y in clusters for x in y]
def get_cluster_status(clus_name, dep_name=None, env_name=None):
env_name = env_name if env_name else horton.cdcred.name
dep_name = dep_name if dep_name else clus_name
#log.info("Fetching Cluster status for [%s]", clus_name)
try:
return cd.ClustersApi(horton.cad).get_status(environment=env_name,
deployment=dep_name,
cluster=clus_name)
except ApiException as e:
if e.status == 404:
log.error("Cluster %s not found", dep_name)
return None
else:
raise e
def get_cluster(clus_name, dep_name=None, env_name=None):
env_name = env_name if env_name else horton.cdcred.name
dep_name = dep_name if dep_name else clus_name
log.info("Attempting to get Cluster %s", clus_name)
try:
return cd.ClustersApi(horton.cad).get_redacted(environment=env_name,
deployment=dep_name,
cluster=clus_name)
except ApiException as e:
if e.status == 404:
log.error("Cluster %s not found, error: %s", clus_name, e.body)
return []
else:
raise e
def create_cluster(cluster_def,
dep_name,
workers=3,
env_name=None,
scripts=None):
env_name = env_name if env_name else horton.cdcred.name
cdh_ver = str(cluster_def['products']['CDH'])
services = cluster_def['services']
cluster_name = cluster_def['name']
products = cluster_def['products']
if 'post_create_scripts' in cluster_def and cluster_def[
'post_create_scripts'] is not None:
log.info("Including post_create_scripts in cluster of %s",
cluster_def['post_create_scripts'])
post_create_scripts = []
for script_name in cluster_def['post_create_scripts']:
log.info("Adding script %s like [%s]", script_name,
scripts[script_name][:50])
post_create_scripts.append(cd.Script(content=scripts[script_name]))
else:
post_create_scripts = None
if cdh_ver[0] == '5':
load_parcels = [
'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/'
]
elif cdh_ver[0] == '6':
load_parcels = [
'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/'
]
else:
raise ValueError("Only CDH versions 5 or 6 supported")
if 'parcels' in cluster_def:
load_parcels += cluster_def['parcels']
# Default Role and Service configs
services_configs = {}
master_setups = {}
master_configs = {}
worker_setups = {}
worker_configs = {}
if 'HDFS' in services:
master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE']
worker_setups['HDFS'] = ['DATANODE']
if 'YARN' in services:
master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY']
worker_setups['YARN'] = ['NODEMANAGER']
if 'ZOOKEEPER' in services:
master_setups['ZOOKEEPER'] = ['SERVER']
if 'HBASE' in services:
master_setups['HBASE'] = ['MASTER']
worker_setups['HBASE'] = ['REGIONSERVER']
if 'HIVE' in services:
master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE']
if 'HUE' in services:
master_setups['HUE'] = ['HUE_SERVER']
if 'KUDU' in services:
master_setups['KUDU'] = ['KUDU_MASTER']
worker_setups['KUDU'] = ['KUDU_TSERVER']
master_configs['KUDU'] = {
'KUDU_MASTER': {
'fs_wal_dir': "/data0/kudu/masterwal",
'fs_data_dirs': "/data1/kudu/master"
}
}
worker_configs['KUDU'] = {
'KUDU_TSERVER': {
'fs_wal_dir': "/data0/kudu/tabletwal",
'fs_data_dirs': "/data1/kudu/tablet"
}
}
if 'IMPALA' in services:
master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE']
worker_setups['IMPALA'] = ['IMPALAD']
if 'NIFI' in services:
worker_setups['NIFI'] = ['NIFI_NODE']
if 'NIFIREGISTRY' in services:
master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER']
if 'NIFITOOLKITCA' in services:
master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER']
services_configs['NIFITOOLKITCA'] = {
'nifi.toolkit.tls.ca.server.token':
security.get_secret('MASTERKEY')
}
if 'KAFKA' in services:
worker_setups['KAFKA'] = ['KAFKA_BROKER']
services_configs['KAFKA'] = {'producer.metrics.enable': True}
if 'SCHEMAREGISTRY' in services:
master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER']
# Handle Services Configs overrides
if 'servicesconfigs' in cluster_def.keys():
for k, v in cluster_def['servicesconfigs']:
services_configs[k] = v
# Handle virtual instance generation
master_vi = [
create_virtual_instance(tem_name=dep_name + '-' + cluster_name +
'-master',
scripts=[
'''sudo -i
yum install mysql mariadb-server epel-release -y # MariaDB
yum -y install npm gcc-c++ make # SMM-UI
npm install forever -g # SMM-UI
systemctl enable mariadb
service mariadb start
mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION"
mysql --execute="CREATE DATABASE streamsmsgmgr DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'streamsmsgmgr'@'localhost' IDENTIFIED BY 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' identified by 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' WITH GRANT OPTION"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' identified by 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' WITH GRANT OPTION"
mysql --execute="FLUSH PRIVILEGES"
mysql --execute="COMMIT"'''
])
]
worker_vi = [
create_virtual_instance(tem_name=dep_name + '-' + cluster_name +
'-worker') for _ in range(0, workers)
]
try:
cd.ClustersApi(horton.cad).create(
environment=env_name,
deployment=dep_name,
cluster_template=cd.ClusterTemplate(
name=cluster_name,
product_versions=products,
parcel_repositories=load_parcels,
services=services,
services_configs=services_configs,
virtual_instance_groups={
'masters':
cd.VirtualInstanceGroup(
name='masters',
min_count=1,
service_type_to_role_types=master_setups,
role_types_configs=master_configs,
virtual_instances=master_vi),
'workers':
cd.VirtualInstanceGroup(
name='workers',
min_count=workers,
service_type_to_role_types=worker_setups,
role_types_configs=worker_configs,
virtual_instances=worker_vi)
},
post_create_scripts=post_create_scripts))
except ApiException as e:
if e.status == 409:
log.error("Cluster %s already exists", cluster_name)
raise ValueError("Cluster %s already exists", cluster_name)
else:
raise e
def create_cluster(cdh_ver,
workers=3,
services=None,
env_name=None,
dep_name=None,
clus_name=None):
env_name = env_name if env_name else horton.cadcred.name
dep_name = dep_name if dep_name else env_name + '-' + cdh_ver.replace(
'.', '-')
services = services if services else ['HDFS', 'YARN']
master_setups = {}
master_configs = {}
worker_setups = {}
worker_configs = {}
if 'HDFS' in services:
master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE']
worker_setups['HDFS'] = ['DATANODE']
if 'YARN' in services:
master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY']
worker_setups['YARN'] = ['NODEMANAGER']
if 'ZOOKEEPER' in services:
master_setups['ZOOKEEPER'] = ['SERVER']
if 'HBASE' in services:
master_setups['HBASE'] = ['MASTER']
worker_setups['HBASE'] = ['REGIONSERVER']
if 'HIVE' in services:
master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE']
if 'HUE' in services:
master_setups['HUE'] = ['HUE_SERVER']
if 'KUDU' in services:
master_setups['KUDU'] = ['KUDU_MASTER']
worker_setups['KUDU'] = ['KUDU_TSERVER']
master_configs['KUDU'] = {
'KUDU_MASTER': {
'fs_wal_dir': "/data0/kudu/masterwal",
'fs_data_dirs': "/data1/kudu/master"
}
}
worker_configs['KUDU'] = {
'KUDU_TSERVER': {
'fs_wal_dir': "/data0/kudu/tabletwal",
'fs_data_dirs': "/data1/kudu/tablet"
}
}
if 'IMPALA' in services:
master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE']
worker_setups['IMPALA'] = ['IMPALAD']
clus_name = clus_name if clus_name else \
horton.cadcred.name + '-' + str(cdh_ver).replace('.', '-')
if cdh_ver[0] == '5':
parcels = [
'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/'
]
elif cdh_ver[0] == '6':
parcels = [
'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/'
]
else:
raise ValueError("Only CDH versions 5 or 6 supported")
try:
cd.ClustersApi(horton.cad).create(
environment=env_name,
deployment=dep_name,
cluster_template=cd.ClusterTemplate(
name=clus_name,
product_versions={'CDH': cdh_ver},
parcel_repositories=parcels,
services=services,
services_configs={},
virtual_instance_groups={
'masters':
cd.VirtualInstanceGroup(
name='masters',
min_count=1,
service_type_to_role_types=master_setups,
role_types_configs=master_configs,
virtual_instances=[create_virtual_instance()]),
'workers':
cd.VirtualInstanceGroup(
name='workers',
min_count=workers,
service_type_to_role_types=worker_setups,
role_types_configs=worker_configs,
virtual_instances=[
create_virtual_instance()
for _ in range(0, workers)
])
}))
except ApiException as e:
if e.status == 409:
log.error("Cluster %s already exists", clus_name)
raise ValueError("Cluster %s already exists", clus_name)
else:
raise e
def list_clusters(env_name=None, dep_name=None):
# Using default if available
env_name = env_name if env_name else horton.cadcred.name
dep_name = dep_name if dep_name else horton.cadcred.name
cd.ClustersApi(horton.cad).list(environment=env_name, deployment=dep_name)
def create_cluster(cdh_ver,
workers=3,
services=None,
env_name=None,
dep_name=None,
clus_name=None,
parcels=None):
env_name = env_name if env_name else horton.cdcred.name
dep_name = dep_name if dep_name else env_name + '-' + cdh_ver.replace(
'.', '-')
services = services if services else ['HDFS', 'YARN']
if cdh_ver[0] == '5':
load_parcels = [
'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/'
]
elif cdh_ver[0] == '6':
load_parcels = [
'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/'
]
else:
raise ValueError("Only CDH versions 5 or 6 supported")
if parcels:
load_parcels += parcels
if cdh_ver[0] == '6':
# CDH6 already has Kafka
load_parcels = [x for x in load_parcels if 'kafka' not in x]
products = {'CDH': cdh_ver}
if 'NIFI' in str(services):
products['CFM'] = '1'
if 'KAFKA' in str(services) and cdh_ver[0] == '5':
products['KAFKA'] = '4'
if 'SCHEMAREGISTRY' in str(services):
products['SCHEMAREGISTRY'] = '0.7'
services_configs = {}
master_setups = {}
master_configs = {}
worker_setups = {}
worker_configs = {}
if 'HDFS' in services:
master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE']
worker_setups['HDFS'] = ['DATANODE']
if 'YARN' in services:
master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY']
worker_setups['YARN'] = ['NODEMANAGER']
if 'ZOOKEEPER' in services:
master_setups['ZOOKEEPER'] = ['SERVER']
if 'HBASE' in services:
master_setups['HBASE'] = ['MASTER']
worker_setups['HBASE'] = ['REGIONSERVER']
if 'HIVE' in services:
master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE']
if 'HUE' in services:
master_setups['HUE'] = ['HUE_SERVER']
if 'KUDU' in services:
master_setups['KUDU'] = ['KUDU_MASTER']
worker_setups['KUDU'] = ['KUDU_TSERVER']
master_configs['KUDU'] = {
'KUDU_MASTER': {
'fs_wal_dir': "/data0/kudu/masterwal",
'fs_data_dirs': "/data1/kudu/master"
}
}
worker_configs['KUDU'] = {
'KUDU_TSERVER': {
'fs_wal_dir': "/data0/kudu/tabletwal",
'fs_data_dirs': "/data1/kudu/tablet"
}
}
if 'IMPALA' in services:
master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE']
worker_setups['IMPALA'] = ['IMPALAD']
if 'NIFI' in services:
worker_setups['NIFI'] = ['NIFI_NODE']
if 'NIFIREGISTRY' in services:
master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER']
if 'NIFITOOLKITCA' in services:
master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER']
services_configs['NIFITOOLKITCA'] = {
'nifi.toolkit.tls.ca.server.token':
security.get_secret('MASTERKEY')
}
if 'KAFKA' in services:
worker_setups['KAFKA'] = ['KAFKA_BROKER']
if 'SCHEMAREGISTRY' in services:
master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER']
clus_name = clus_name if clus_name else \
horton.cdcred.name + '-' + str(cdh_ver).replace('.', '-')
# Handle virtual instance generation
master_vi = [
create_virtual_instance(tem_name='master',
scripts=[
'''sudo -i
yum install mysql mariadb-server -y
systemctl enable mariadb
service mariadb start
mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION"
mysql --execute="FLUSH PRIVILEGES"
mysql --execute="COMMIT"'''
])
]
worker_vi = [create_virtual_instance() for _ in range(0, workers)]
try:
cd.ClustersApi(horton.cad).create(
environment=env_name,
deployment=dep_name,
cluster_template=cd.ClusterTemplate(
name=clus_name,
product_versions=products,
parcel_repositories=load_parcels,
services=services,
services_configs=services_configs,
virtual_instance_groups={
'masters':
cd.VirtualInstanceGroup(
name='masters',
min_count=1,
service_type_to_role_types=master_setups,
role_types_configs=master_configs,
virtual_instances=master_vi),
'workers':
cd.VirtualInstanceGroup(
name='workers',
min_count=workers,
service_type_to_role_types=worker_setups,
role_types_configs=worker_configs,
virtual_instances=worker_vi)
}))
except ApiException as e:
if e.status == 409:
log.error("Cluster %s already exists", clus_name)
raise ValueError("Cluster %s already exists", clus_name)
else:
raise e
