def _create_services(cluster): api = cu.get_api_client(cluster) fullversion = ('5.0.0' if cluster.hadoop_version == '5' else cluster.hadoop_version) cm_cluster = api.create_cluster(cluster.name, fullVersion=fullversion) if len(pu.get_zookeepers(cluster)) > 0: cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME, ZOOKEEPER_SERVICE_TYPE) cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) if pu.get_hive_metastore(cluster): cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE) if pu.get_hue(cluster): cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) if pu.get_spark_historyserver(cluster): cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE) if pu.get_hbase_master(cluster): cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE) if len(pu.get_flumes(cluster)) > 0: cm_cluster.create_service(cu.FLUME_SERVICE_NAME, FLUME_SERVICE_TYPE) if pu.get_sentry(cluster): cm_cluster.create_service(cu.SENTRY_SERVICE_NAME, SENTRY_SERVICE_TYPE) if len(pu.get_solrs(cluster)) > 0: cm_cluster.create_service(cu.SOLR_SERVICE_NAME, SOLR_SERVICE_TYPE) if pu.get_sqoop(cluster): cm_cluster.create_service(cu.SQOOP_SERVICE_NAME, SQOOP_SERVICE_TYPE) if len(pu.get_hbase_indexers(cluster)) > 0: cm_cluster.create_service(cu.KS_INDEXER_SERVICE_NAME, KS_INDEXER_SERVICE_TYPE) if pu.get_catalogserver(cluster): cm_cluster.create_service(cu.IMPALA_SERVICE_NAME, IMPALA_SERVICE_TYPE)
def start_cluster(cluster): cm_cluster = cu.get_cloudera_cluster(cluster) hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME) cu.format_namenode(hdfs) cu.start_service(hdfs) yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME) cu.create_yarn_job_history_dir(yarn) cu.start_service(yarn) oozie_inst = pu.get_oozie(cluster) if oozie_inst: _install_extjs(cluster) oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME) cu.create_oozie_db(oozie) cu.install_oozie_sharelib(oozie) cu.start_service(oozie) if pu.get_hive_metastore(cluster): hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME) _configure_hive(cluster) cu.create_hive_metastore_db(hive) cu.create_hive_dirs(hive) cu.start_service(hive) if pu.get_hue(cluster): hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME) cu.start_service(hue) if pu.get_spark_historyserver(cluster): _configure_spark(cluster) spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME) cu.start_service(spark)
def _configure_services(cluster): cm_cluster = cu.get_cloudera_cluster(cluster) if len(pu.get_zookeepers(cluster)) > 0: zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME) zookeeper.update_config( _get_configs(ZOOKEEPER_SERVICE_TYPE, cluster=cluster)) hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME) hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster)) yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME) yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster)) oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME) oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster)) if pu.get_hive_metastore(cluster): hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME) hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster)) if pu.get_hue(cluster): hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME) hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster)) if pu.get_spark_historyserver(cluster): spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME) spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster)) if pu.get_hbase_master(cluster): hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME) hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster))
def _configure_services(cluster): cm_cluster = cu.get_cloudera_cluster(cluster) if len(pu.get_zookeepers(cluster)) > 0: zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME) zookeeper.update_config( _get_configs(ZOOKEEPER_SERVICE_TYPE, cluster=cluster)) hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME) hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster)) yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME) yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster)) oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME) oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster)) if pu.get_hive_metastore(cluster): hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME) hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster)) if pu.get_hue(cluster): hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME) hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster)) if pu.get_spark_historyserver(cluster): spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME) spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster)) if pu.get_hbase_master(cluster): hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME) hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster)) if len(pu.get_flumes(cluster)) > 0: flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME) flume.update_config(_get_configs(FLUME_SERVICE_TYPE, cluster=cluster)) if pu.get_sentry(cluster): sentry = cm_cluster.get_service(cu.SENTRY_SERVICE_NAME) sentry.update_config(_get_configs(SENTRY_SERVICE_TYPE, cluster=cluster)) if len(pu.get_solrs(cluster)) > 0: solr = cm_cluster.get_service(cu.SOLR_SERVICE_NAME) solr.update_config(_get_configs(SOLR_SERVICE_TYPE, cluster=cluster)) if pu.get_sqoop(cluster): sqoop = cm_cluster.get_service(cu.SQOOP_SERVICE_NAME) sqoop.update_config(_get_configs(SQOOP_SERVICE_TYPE, cluster=cluster)) if len(pu.get_hbase_indexers(cluster)) > 0: ks_indexer = cm_cluster.get_service(cu.KS_INDEXER_SERVICE_NAME) ks_indexer.update_config( _get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster)) if pu.get_catalogserver(cluster): impala = cm_cluster.get_service(cu.IMPALA_SERVICE_NAME) impala.update_config(_get_configs(IMPALA_SERVICE_TYPE, cluster=cluster))
def _configure_services(cluster): cm_cluster = cu.get_cloudera_cluster(cluster) if len(pu.get_zookeepers(cluster)) > 0: zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME) zookeeper.update_config(_get_configs(ZOOKEEPER_SERVICE_TYPE, cluster=cluster)) hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME) hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster)) yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME) yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster)) oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME) oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster)) if pu.get_hive_metastore(cluster): hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME) hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster)) if pu.get_hue(cluster): hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME) hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster)) if pu.get_spark_historyserver(cluster): spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME) spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster)) if pu.get_hbase_master(cluster): hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME) hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster)) if len(pu.get_flumes(cluster)) > 0: flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME) flume.update_config(_get_configs(FLUME_SERVICE_TYPE, cluster=cluster)) if pu.get_sentry(cluster): sentry = cm_cluster.get_service(cu.SENTRY_SERVICE_NAME) sentry.update_config(_get_configs(SENTRY_SERVICE_TYPE, cluster=cluster)) if len(pu.get_solrs(cluster)) > 0: solr = cm_cluster.get_service(cu.SOLR_SERVICE_NAME) solr.update_config(_get_configs(SOLR_SERVICE_TYPE, cluster=cluster)) if pu.get_sqoop(cluster): sqoop = cm_cluster.get_service(cu.SQOOP_SERVICE_NAME) sqoop.update_config(_get_configs(SQOOP_SERVICE_TYPE, cluster=cluster)) if len(pu.get_hbase_indexers(cluster)) > 0: ks_indexer = cm_cluster.get_service(cu.KS_INDEXER_SERVICE_NAME) ks_indexer.update_config(_get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster)) if pu.get_catalogserver(cluster): impala = cm_cluster.get_service(cu.IMPALA_SERVICE_NAME) impala.update_config(_get_configs(IMPALA_SERVICE_TYPE, cluster=cluster))
def _create_services(cluster): api = cu.get_api_client(cluster) cm_cluster = api.create_cluster(cluster.name, CDH_VERSION) cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) if pu.get_hive_metastore(cluster): cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE) if pu.get_hue(cluster): cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) if pu.get_spark_historyserver(cluster): cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
def _set_cluster_info(self, cluster): mng = cu.get_manager(cluster) info = { 'Cloudera Manager': { 'Web UI': 'http://%s:7180' % mng.management_ip, 'Username': '******', 'Password': '******' } } hue = cu.get_hue(cluster) if hue: info['Hue Dashboard'] = { 'Web UI': 'http://%s:8888' % hue.management_ip } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster): mng = cu.get_manager(cluster) info = { 'Cloudera Manager': { 'Web UI': 'http://%s:7180' % mng.management_ip, 'Username': '******', 'Password': '******' } } hue = cu.get_hue(cluster) if hue: info['Hue Dashboard'] = { 'Web UI': 'http://%s:8888' % hue.management_ip } ctx = context.ctx() conductor.cluster_update(ctx, cluster, {'info': info})
def _create_services(cluster): api = cu.get_api_client(cluster) fullversion = ('5.0.0' if cluster.hadoop_version == '5' else cluster.hadoop_version) cm_cluster = api.create_cluster(cluster.name, fullVersion=fullversion) if len(pu.get_zookeepers(cluster)) > 0: cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME, ZOOKEEPER_SERVICE_TYPE) cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) if pu.get_hive_metastore(cluster): cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE) if pu.get_hue(cluster): cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) if pu.get_spark_historyserver(cluster): cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE) if pu.get_hbase_master(cluster): cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE)
def _configure_services(cluster): cm_cluster = cu.get_cloudera_cluster(cluster) hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME) hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster)) yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME) yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster)) oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME) oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster)) if pu.get_hive_metastore(cluster): hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME) hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster)) if pu.get_hue(cluster): hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME) hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster)) if pu.get_spark_historyserver(cluster): spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME) spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
def _create_services(cluster): api = cu.get_api_client(cluster) fullversion = ('5.0.0' if cluster.hadoop_version == '5' else cluster.hadoop_version) cm_cluster = api.create_cluster(cluster.name, fullVersion=fullversion) if len(pu.get_zookeepers(cluster)) > 0: cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME, ZOOKEEPER_SERVICE_TYPE) cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) if pu.get_hive_metastore(cluster): cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE) if pu.get_hue(cluster): cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) if pu.get_spark_historyserver(cluster): cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE) if pu.get_hbase_master(cluster): cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE) if len(pu.get_flumes(cluster)) > 0: cm_cluster.create_service(cu.FLUME_SERVICE_NAME, FLUME_SERVICE_TYPE) if pu.get_sentry(cluster): cm_cluster.create_service(cu.SENTRY_SERVICE_NAME, SENTRY_SERVICE_TYPE) if len(pu.get_solrs(cluster)) > 0: cm_cluster.create_service(cu.SOLR_SERVICE_NAME, SOLR_SERVICE_TYPE) if pu.get_sqoop(cluster): cm_cluster.create_service(cu.SQOOP_SERVICE_NAME, SQOOP_SERVICE_TYPE) if len(pu.get_hbase_indexers(cluster)) > 0: cm_cluster.create_service(cu.KS_INDEXER_SERVICE_NAME, KS_INDEXER_SERVICE_TYPE) if pu.get_catalogserver(cluster): cm_cluster.create_service(cu.IMPALA_SERVICE_NAME, IMPALA_SERVICE_TYPE)