def _create_services(cluster):
api = cu.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5' else
cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name, fullVersion=fullversion)
if len(pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if pu.get_hive_metastore(cluster):
cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE)
if pu.get_hue(cluster):
cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if pu.get_spark_historyserver(cluster):
cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
if pu.get_hbase_master(cluster):
cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE)
if len(pu.get_flumes(cluster)) > 0:
cm_cluster.create_service(cu.FLUME_SERVICE_NAME, FLUME_SERVICE_TYPE)
if pu.get_sentry(cluster):
cm_cluster.create_service(cu.SENTRY_SERVICE_NAME, SENTRY_SERVICE_TYPE)
if len(pu.get_solrs(cluster)) > 0:
cm_cluster.create_service(cu.SOLR_SERVICE_NAME, SOLR_SERVICE_TYPE)
if pu.get_sqoop(cluster):
cm_cluster.create_service(cu.SQOOP_SERVICE_NAME, SQOOP_SERVICE_TYPE)
if len(pu.get_hbase_indexers(cluster)) > 0:
cm_cluster.create_service(cu.KS_INDEXER_SERVICE_NAME,
KS_INDEXER_SERVICE_TYPE)
if pu.get_catalogserver(cluster):
cm_cluster.create_service(cu.IMPALA_SERVICE_NAME, IMPALA_SERVICE_TYPE)
def start_cluster(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
cu.format_namenode(hdfs)
cu.start_service(hdfs)
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
cu.create_yarn_job_history_dir(yarn)
cu.start_service(yarn)
oozie_inst = pu.get_oozie(cluster)
if oozie_inst:
_install_extjs(cluster)
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
cu.create_oozie_db(oozie)
cu.install_oozie_sharelib(oozie)
cu.start_service(oozie)
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
_configure_hive(cluster)
cu.create_hive_metastore_db(hive)
cu.create_hive_dirs(hive)
cu.start_service(hive)
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
cu.start_service(hue)
if pu.get_spark_historyserver(cluster):
_configure_spark(cluster)
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
cu.start_service(spark)
def _configure_services(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
if len(pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(
_get_configs(ZOOKEEPER_SERVICE_TYPE, cluster=cluster))
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster))
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster))
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster))
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster))
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster))
if pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
if pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME)
hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster))
def _configure_services(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
if len(pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(
_get_configs(ZOOKEEPER_SERVICE_TYPE, cluster=cluster))
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster))
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster))
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster))
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster))
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster))
if pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
if pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME)
hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster))
if len(pu.get_flumes(cluster)) > 0:
flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME)
flume.update_config(_get_configs(FLUME_SERVICE_TYPE, cluster=cluster))
if pu.get_sentry(cluster):
sentry = cm_cluster.get_service(cu.SENTRY_SERVICE_NAME)
sentry.update_config(_get_configs(SENTRY_SERVICE_TYPE,
cluster=cluster))
if len(pu.get_solrs(cluster)) > 0:
solr = cm_cluster.get_service(cu.SOLR_SERVICE_NAME)
solr.update_config(_get_configs(SOLR_SERVICE_TYPE, cluster=cluster))
if pu.get_sqoop(cluster):
sqoop = cm_cluster.get_service(cu.SQOOP_SERVICE_NAME)
sqoop.update_config(_get_configs(SQOOP_SERVICE_TYPE, cluster=cluster))
if len(pu.get_hbase_indexers(cluster)) > 0:
ks_indexer = cm_cluster.get_service(cu.KS_INDEXER_SERVICE_NAME)
ks_indexer.update_config(
_get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster))
if pu.get_catalogserver(cluster):
impala = cm_cluster.get_service(cu.IMPALA_SERVICE_NAME)
impala.update_config(_get_configs(IMPALA_SERVICE_TYPE,
cluster=cluster))
def _configure_services(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
if len(pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(_get_configs(ZOOKEEPER_SERVICE_TYPE,
cluster=cluster))
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster))
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster))
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster))
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster))
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster))
if pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
if pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME)
hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster))
if len(pu.get_flumes(cluster)) > 0:
flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME)
flume.update_config(_get_configs(FLUME_SERVICE_TYPE, cluster=cluster))
if pu.get_sentry(cluster):
sentry = cm_cluster.get_service(cu.SENTRY_SERVICE_NAME)
sentry.update_config(_get_configs(SENTRY_SERVICE_TYPE,
cluster=cluster))
if len(pu.get_solrs(cluster)) > 0:
solr = cm_cluster.get_service(cu.SOLR_SERVICE_NAME)
solr.update_config(_get_configs(SOLR_SERVICE_TYPE, cluster=cluster))
if pu.get_sqoop(cluster):
sqoop = cm_cluster.get_service(cu.SQOOP_SERVICE_NAME)
sqoop.update_config(_get_configs(SQOOP_SERVICE_TYPE, cluster=cluster))
if len(pu.get_hbase_indexers(cluster)) > 0:
ks_indexer = cm_cluster.get_service(cu.KS_INDEXER_SERVICE_NAME)
ks_indexer.update_config(_get_configs(KS_INDEXER_SERVICE_TYPE,
cluster=cluster))
if pu.get_catalogserver(cluster):
impala = cm_cluster.get_service(cu.IMPALA_SERVICE_NAME)
impala.update_config(_get_configs(IMPALA_SERVICE_TYPE,
cluster=cluster))
def _create_services(cluster):
api = cu.get_api_client(cluster)
cm_cluster = api.create_cluster(cluster.name, CDH_VERSION)
cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if pu.get_hive_metastore(cluster):
cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE)
if pu.get_hue(cluster):
cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if pu.get_spark_historyserver(cluster):
cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
def _set_cluster_info(self, cluster):
mng = cu.get_manager(cluster)
info = {
'Cloudera Manager': {
'Web UI': 'http://%s:7180' % mng.management_ip,
'Username': '******',
'Password': '******'
}
}
hue = cu.get_hue(cluster)
if hue:
info['Hue Dashboard'] = {
'Web UI': 'http://%s:8888' % hue.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster):
mng = cu.get_manager(cluster)
info = {
'Cloudera Manager': {
'Web UI': 'http://%s:7180' % mng.management_ip,
'Username': '******',
'Password': '******'
}
}
hue = cu.get_hue(cluster)
if hue:
info['Hue Dashboard'] = {
'Web UI': 'http://%s:8888' % hue.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _create_services(cluster):
api = cu.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5' else
cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name, fullVersion=fullversion)
if len(pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if pu.get_hive_metastore(cluster):
cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE)
if pu.get_hue(cluster):
cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if pu.get_spark_historyserver(cluster):
cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
if pu.get_hbase_master(cluster):
cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE)
def _configure_services(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster))
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster))
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster))
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster))
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster))
if pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
def _create_services(cluster):
api = cu.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5'
else cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name,
fullVersion=fullversion)
if len(pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if pu.get_hive_metastore(cluster):
cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE)
if pu.get_hue(cluster):
cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if pu.get_spark_historyserver(cluster):
cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
if pu.get_hbase_master(cluster):
cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE)
if len(pu.get_flumes(cluster)) > 0:
cm_cluster.create_service(cu.FLUME_SERVICE_NAME, FLUME_SERVICE_TYPE)
if pu.get_sentry(cluster):
cm_cluster.create_service(cu.SENTRY_SERVICE_NAME, SENTRY_SERVICE_TYPE)
if len(pu.get_solrs(cluster)) > 0:
cm_cluster.create_service(cu.SOLR_SERVICE_NAME,
SOLR_SERVICE_TYPE)
if pu.get_sqoop(cluster):
cm_cluster.create_service(cu.SQOOP_SERVICE_NAME, SQOOP_SERVICE_TYPE)
if len(pu.get_hbase_indexers(cluster)) > 0:
cm_cluster.create_service(cu.KS_INDEXER_SERVICE_NAME,
KS_INDEXER_SERVICE_TYPE)
if pu.get_catalogserver(cluster):
cm_cluster.create_service(cu.IMPALA_SERVICE_NAME, IMPALA_SERVICE_TYPE)