def test_get_service_name_by_node_process(self): ctx = self._get_context() s_name_1 = ctx.get_service_name_by_node_process(yarn.RESOURCE_MANAGER) self.assertEqual(yarn.YARN().ui_name, s_name_1) s_name_2 = ctx.get_service_name_by_node_process( yarn.RESOURCE_MANAGER.ui_name) self.assertEqual(yarn.YARN().ui_name, s_name_2) not_existing_np = np.NodeProcess('not_existing', 'NotExisting', 'foo') self.assertIsNone( ctx.get_service_name_by_node_process(not_existing_np)) self.assertIsNone( ctx.get_service_name_by_node_process(not_existing_np.ui_name))
# not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu PIG = np.NodeProcess(name='pig', ui_name='Pig', package='mapr-pig') class Pig(s.Service): def __init__(self): super(Pig, self).__init__() self._name = 'pig' self._ui_name = 'Pig' self._node_processes = [PIG] self._validation_rules = [vu.at_least(1, PIG)] class PigV013(Pig): def __init__(self): super(PigV013, self).__init__() self._version = '0.13'
import sahara_plugins.plugins.mapr.services.mapreduce.mapreduce as mr import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql import sahara_plugins.plugins.mapr.services.oozie.oozie as oozie import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry import sahara_plugins.plugins.mapr.services.spark.spark as spark import sahara_plugins.plugins.mapr.services.sqoop.sqoop2 as sqoop import sahara_plugins.plugins.mapr.services.yarn.yarn as yarn import sahara_plugins.plugins.mapr.util.event_log as el import sahara_plugins.plugins.mapr.util.general as g import sahara_plugins.plugins.mapr.util.password_utils as pu import sahara_plugins.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) HUE = np.NodeProcess(name='hue', ui_name='Hue', package='mapr-hue', open_ports=[8002, 8888]) HUE_LIVY = np.NodeProcess(name="livy", ui_name="Hue Livy", package="mapr-hue-livy", open_ports=[8998]) class Hue(s.Service): THRIFT_VERSIONS = [5, 7] THRIFT_VERSION = p.Config(name="Thrift version", applicable_target="Hue", scope='cluster', config_type="dropdown", config_values=[(v, v) for v in THRIFT_VERSIONS],
def __init__(self, *args, **kwds): super(TestClusterContext, self).__init__(*args, **kwds) self.fake_np = np.NodeProcess('fake', 'foo', 'bar')
from sahara.plugins import context import sahara.plugins.provisioning as p from sahara.plugins import utils from sahara_plugins.i18n import _ import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.event_log as el import sahara_plugins.plugins.mapr.util.general as g import sahara_plugins.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) CLDB = np.NodeProcess( name='cldb', ui_name='CLDB', package='mapr-cldb', open_ports=[7222, 7220, 7221] ) FILE_SERVER = np.NodeProcess( name='fileserver', ui_name='FileServer', package='mapr-fileserver', open_ports=[] ) NFS = np.NodeProcess( name='nfs', ui_name='NFS', package='mapr-nfs', open_ports=[2049, 9997, 9998] )
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.commands as cmd import sahara_plugins.plugins.mapr.util.validation_utils as vu DRILL = np.NodeProcess(name='drill-bits', ui_name='Drill', package='mapr-drill', open_ports=[8047]) DRILL_YARN = np.NodeProcess(name='drill-bits-yarn', ui_name='Drill on YARN', package='mapr-drill-yarn', open_ports=[8047]) class Drill(s.Service): def __init__(self): super(Drill, self).__init__() self._name = 'drill' self._ui_name = 'Drill' self._node_processes = [DRILL] self._ui_info = [('Drill', DRILL, {s.SERVICE_UI: 'http://%s:8047'})]
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu FLUME = np.NodeProcess(name='flume', ui_name='Flume', package='mapr-flume', open_ports=[]) class Flume(s.Service): def __init__(self): super(Flume, self).__init__() self._name = 'flume' self._ui_name = 'Flume' self._node_processes = [FLUME] self._validation_rules = [vu.at_least(1, FLUME)] class FlumeV15(Flume): def __init__(self): super(FlumeV15, self).__init__()
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu HTTP_FS = np.NodeProcess(name='httpfs', ui_name='HTTPFS', package='mapr-httpfs', open_ports=[14000]) class HttpFS(s.Service): def __init__(self): super(HttpFS, self).__init__() self._name = 'httpfs' self._ui_name = 'HttpFS' self._version = '1.0' self._node_processes = [HTTP_FS] self._cluster_defaults = ['httpfs-default.json'] self._validation_rules = [vu.exactly(1, HTTP_FS)] def post_install(self, cluster_context, instances): self._set_service_dir_owner(cluster_context, instances)
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.utils as utils import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.services.hbase.hbase as hbase import sahara_plugins.plugins.mapr.services.hive.hive as hive import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry import sahara_plugins.plugins.mapr.util.general as g import sahara_plugins.plugins.mapr.util.validation_utils as vu IMPALA_SERVER = np.NodeProcess(name='impalaserver', ui_name='Impala-Server', package='mapr-impala-server', open_ports=[21000, 21050, 25000]) IMPALA_STATE_STORE = np.NodeProcess(name='impalastore', ui_name='Impala-Statestore', package='mapr-impala-statestore', open_ports=[25010]) IMPALA_CATALOG = np.NodeProcess(name='impalacatalog', ui_name='Impala-Catalog', package='mapr-impala-catalog', open_ports=[25020]) class Impala(s.Service): def __init__(self): super(Impala, self).__init__() self._name = 'impala'
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.password_utils as pu import sahara_plugins.plugins.mapr.util.validation_utils as vu ZK_CLIENT_PORT = 5181 ZOOKEEPER = np.NodeProcess(name='mapr-zookeeper', ui_name='ZooKeeper', package='mapr-zookeeper', open_ports=[ZK_CLIENT_PORT]) WEB_SERVER = np.NodeProcess(name='webserver', ui_name='Webserver', package='mapr-webserver', open_ports=[8443]) METRICS = np.NodeProcess(name='metrics', ui_name='Metrics', package='mapr-metrics', open_ports=[1111]) class Management(s.Service): def __init__(self):
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from sahara.plugins import swift_helper from sahara.plugins import topology_helper as topo import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu JOB_TRACKER = np.NodeProcess(name='jobtracker', ui_name='JobTracker', package='mapr-jobtracker', open_ports=[9001, 50030]) TASK_TRACKER = np.NodeProcess(name='tasktracker', ui_name='TaskTracker', package='mapr-tasktracker', open_ports=[50060]) class MapReduce(s.Service): cluster_mode = 'classic' def __init__(self): super(MapReduce, self).__init__() self._ui_name = 'MapReduce' self._name = 'hadoop' self._version = '0.20.2'
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu HBASE_MASTER = np.NodeProcess( name='hbmaster', ui_name='HBase-Master', package='mapr-hbase-master', open_ports=[60000, 60010] ) HBASE_REGION_SERVER = np.NodeProcess( name='hbregionserver', ui_name='HBase-RegionServer', package='mapr-hbase-regionserver', open_ports=[60020] ) HBASE_THRIFT = np.NodeProcess( name='hbasethrift', ui_name='HBase-Thrift', package='mapr-hbasethrift', open_ports=[9090] ) HBASE_REST = np.NodeProcess(
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu KAFKA = np.NodeProcess(name='kafka', ui_name='Kafka', package='mapr-kafka', open_ports=[9092]) KAFKA_REST = np.NodeProcess(name='kafka', ui_name='Kafka Rest', package='mapr-kafka-rest', open_ports=[8082]) KAFKA_CONNECT_HDFS = np.NodeProcess(name='kafka', ui_name='Kafka Connect HDFS', package='mapr-kafka-connect-hdfs') KAFKA_CONNECT_JDBC = np.NodeProcess(name='kafka', ui_name='Kafka Connect JDBC', package='mapr-kafka-connect-jdbc')
import sahara.plugins.context as context from sahara_plugins.i18n import _ import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql import sahara_plugins.plugins.mapr.util.event_log as el import sahara_plugins.plugins.mapr.util.general as g import sahara_plugins.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) OOZIE_START_DELAY = 30 OOZIE = np.NodeProcess( name='oozie', ui_name='Oozie', package='mapr-oozie', open_ports=[11000] ) class Oozie(s.Service): def __init__(self): super(Oozie, self).__init__() self._name = 'oozie' self._ui_name = 'Oozie' self._node_processes = [OOZIE] self._cluster_defaults = ['oozie-default.json'] self._validation_rules = [vu.exactly(1, OOZIE)] self._ui_info = [('Oozie', OOZIE, {s.SERVICE_UI: 'http://%s:11000/oozie'})]
import sahara.plugins.utils as utils import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.services.hbase.hbase as hbase import sahara_plugins.plugins.mapr.services.hive.hive as hive import sahara_plugins.plugins.mapr.util.general as g import sahara_plugins.plugins.mapr.util.maprfs_helper as mfs import sahara_plugins.plugins.mapr.util.validation_utils as vu SPARK_SLAVE_UI_PORT = 8081 SPARK_HS_UI_PORT = 18080 SPARK_HISTORY_SERVER = np.NodeProcess(name='spark-historyserver', ui_name='Spark HistoryServer', package='mapr-spark-historyserver', open_ports=[SPARK_HS_UI_PORT]) SPARK_SLAVE = np.NodeProcess(name='spark-master', ui_name='Spark Slave', package='mapr-spark', open_ports=[SPARK_SLAVE_UI_PORT]) class SparkOnYarn(s.Service): JAR_FILE_TARGET = '/apps/spark/lib' MFS_DIR = '/apps/spark' SERVLET_JAR = 'javax.servlet-api.jar' def __init__(self): super(SparkOnYarn, self).__init__() self._name = 'spark'
from oslo_log import log as logging from sahara.i18n import _ import sahara_plugins.plugins.mapr.domain.configuration_file as bcf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry import sahara_plugins.plugins.mapr.util.event_log as el import sahara_plugins.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files LOG = logging.getLogger(__name__) HIVE_METASTORE = np.NodeProcess(name='hivemeta', ui_name='HiveMetastore', package='mapr-hivemetastore', open_ports=[9083]) HIVE_SERVER_2 = np.NodeProcess(name='hs2', ui_name='HiveServer2', package='mapr-hiveserver2', open_ports=[10000]) class Hive(s.Service): def __init__(self): super(Hive, self).__init__() self._name = 'hive' self._ui_name = 'Hive' self._node_processes = [HIVE_METASTORE, HIVE_SERVER_2] self._validation_rules = [ vu.exactly(1, HIVE_METASTORE),
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu SQOOP_2_SERVER = np.NodeProcess(name='sqoop2', ui_name='Sqoop2-Server', package='mapr-sqoop2-server', open_ports=[12000]) SQOOP_2_CLIENT = np.NodeProcess(name='sqoop-client', ui_name='Sqoop2-Client', package='mapr-sqoop2-client') class Sqoop2(s.Service): def __init__(self): super(Sqoop2, self).__init__() self._name = 'sqoop' self._ui_name = 'Sqoop2' self._version = '2.0.0' self._node_processes = [SQOOP_2_CLIENT, SQOOP_2_SERVER] self._validation_rules = [ vu.exactly(1, SQOOP_2_SERVER),
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.util.validation_utils as vu MAHOUT = np.NodeProcess(name='mahout', ui_name='Mahout', package='mapr-mahout', open_ports=[]) class Mahout(s.Service): def __init__(self): super(Mahout, self).__init__() self._name = 'mahout' self._ui_name = 'Mahout' self._node_processes = [MAHOUT] self._validation_rules = [vu.at_least(1, MAHOUT)] class MahoutV09(Mahout): def __init__(self): super(MahoutV09, self).__init__()
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.provisioning as p import sahara.plugins.utils as utils from sahara_plugins.i18n import _ import sahara_plugins.plugins.mapr.domain.configuration_file as cf import sahara_plugins.plugins.mapr.domain.node_process as np import sahara_plugins.plugins.mapr.domain.service as s import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql import sahara_plugins.plugins.mapr.util.event_log as el import sahara_plugins.plugins.mapr.util.maprfs_helper as mfs SENTRY = np.NodeProcess(name='sentry', ui_name='Sentry', package='mapr-sentry') SENTRY_MODE_CONFIG_NAME = 'Sentry storage mode' FILE_STORAGE_SENTRY_MODE = 'File-base storage' DB_STORAGE_SENTRY_MODE = 'DB-based storage' class Sentry(s.Service): SENTRY_STORAGE_MODE = p.Config( name=SENTRY_MODE_CONFIG_NAME, applicable_target='Sentry', scope='cluster', config_type="dropdown", config_values=[ (v, v) for v in (FILE_STORAGE_SENTRY_MODE, DB_STORAGE_SENTRY_MODE) ],