def test_get_service_name_by_node_process(self): ctx = self._get_context() s_name_1 = ctx.get_service_name_by_node_process(yarn.RESOURCE_MANAGER) self.assertEqual(yarn.YARN().ui_name, s_name_1) s_name_2 = ctx.get_service_name_by_node_process( yarn.RESOURCE_MANAGER.ui_name) self.assertEqual(yarn.YARN().ui_name, s_name_2) not_existing_np = np.NodeProcess('not_existing', 'NotExisting', 'foo') self.assertIsNone( ctx.get_service_name_by_node_process(not_existing_np)) self.assertIsNone( ctx.get_service_name_by_node_process(not_existing_np.ui_name))
# License for the specific language governing permissions and limitations # under the License. from oslo_log import log as logging import six import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files LOG = logging.getLogger(__name__) HIVE_METASTORE = np.NodeProcess(name='hivemeta', ui_name='HiveMetastore', package='mapr-hivemetastore', open_ports=[9083]) HIVE_SERVER_2 = np.NodeProcess(name='hs2', ui_name='HiveServer2', package='mapr-hiveserver2', open_ports=[10000]) class Hive(s.Service): def __init__(self): super(Hive, self).__init__() self._name = 'hive' self._ui_name = 'Hive' self._node_processes = [HIVE_METASTORE, HIVE_SERVER_2] self._validation_rules = [ vu.exactly(1, HIVE_METASTORE),
import sahara.context as context from sahara.i18n import _ import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.services.mysql.mysql as mysql import sahara.plugins.mapr.util.event_log as el import sahara.plugins.mapr.util.general as g import sahara.plugins.mapr.util.validation_utils as vu LOG = logging.getLogger(__name__) OOZIE_START_DELAY = 30 OOZIE = np.NodeProcess(name='oozie', ui_name='Oozie', package='mapr-oozie', open_ports=[11000]) class Oozie(s.Service): def __init__(self): super(Oozie, self).__init__() self._name = 'oozie' self._ui_name = 'Oozie' self._node_processes = [OOZIE] self._cluster_defaults = ['oozie-default.json'] self._validation_rules = [vu.exactly(1, OOZIE)] self._ui_info = [('Oozie', OOZIE, { s.SERVICE_UI: 'http://%s:11000/oozie' })]
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.services.hive.hive as hive import sahara.plugins.mapr.util.commands as cmd import sahara.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files IMPALA_SERVER = np.NodeProcess(name='impalaserver', ui_name='Impala-Server', package='mapr-impala-server', open_ports=[21000, 21050, 25000]) IMPALA_STATE_STORE = np.NodeProcess(name='impalastore', ui_name='Impala-Statestore', package='mapr-impala-statestore', open_ports=[25010]) IMPALA_CATALOG = np.NodeProcess(name='impalacatalog', ui_name='Impala-Catalog', package='mapr-impala-catalog', open_ports=[25020]) class Impala(s.Service): def __init__(self): super(Impala, self).__init__() self._name = 'impala'
import sahara.plugins.mapr.services.impala.impala as impala import sahara.plugins.mapr.services.mapreduce.mapreduce as mr import sahara.plugins.mapr.services.mysql.mysql as mysql import sahara.plugins.mapr.services.oozie.oozie as oozie import sahara.plugins.mapr.services.sqoop.sqoop2 as sqoop import sahara.plugins.mapr.services.yarn.yarn as yarn import sahara.plugins.mapr.util.general as g import sahara.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files LOG = logging.getLogger(__name__) HUE = np.NodeProcess( name='hue', ui_name='Hue', package='mapr-hue', open_ports=[8002, 8888] ) @six.add_metaclass(s.Single) class Hue(s.Service): def __init__(self): super(Hue, self).__init__() self._name = 'hue' self._ui_name = 'Hue' self._version = '3.6.0' self._node_processes = [HUE] self._ui_info = [('HUE', HUE, 'http://%s:8888')] self._validation_rules = [ vu.exactly(1, HUE),
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import six import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.commands as cmd import sahara.plugins.mapr.util.validation_utils as vu SQOOP_2_SERVER = np.NodeProcess(name='sqoop2', ui_name='Sqoop2-Server', package='mapr-sqoop2-server', open_ports=[12000]) SQOOP_2_CLIENT = np.NodeProcess(name='sqoop-client', ui_name='Sqoop2-Client', package='mapr-sqoop2-client') @six.add_metaclass(s.Single) class Sqoop2(s.Service): def __init__(self): super(Sqoop2, self).__init__() self._name = 'sqoop' self._ui_name = 'Sqoop2' self._version = '2.0.0' self._node_processes = [SQOOP_2_CLIENT, SQOOP_2_SERVER] self._validation_rules = [
def __init__(self, *args, **kwds): super(TestClusterContext, self).__init__(*args, **kwds) self.fake_np = np.NodeProcess('fake', 'foo', 'bar')
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.commands as cmd import sahara.plugins.mapr.util.validation_utils as vu DRILL = np.NodeProcess( name='drill-bits', ui_name='Drill', package='mapr-drill', open_ports=[] ) class Drill(s.Service): def __init__(self): super(Drill, self).__init__() self._name = 'drill' self._ui_name = 'Drill' self._node_processes = [DRILL] self._ui_info = [('Drill', DRILL, 'http://%s:8047')] self._validation_rules = [vu.at_least(1, DRILL)] def install(self, cluster_context, instances): # Drill requires running cluster
import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.services.hbase.hbase as hbase import sahara.plugins.mapr.services.hive.hive as hive import sahara.plugins.mapr.util.general as g import sahara.plugins.mapr.util.maprfs_helper as mfs import sahara.plugins.mapr.util.validation_utils as vu import sahara.utils.files as files SPARK_SLAVE_UI_PORT = 8081 SPARK_HS_UI_PORT = 18080 SPARK_HISTORY_SERVER = np.NodeProcess(name='spark-historyserver', ui_name='Spark HistoryServer', package='mapr-spark-historyserver', open_ports=[SPARK_HS_UI_PORT]) SPARK_SLAVE = np.NodeProcess(name='spark-master', ui_name='Spark Slave', package='mapr-spark', open_ports=[SPARK_SLAVE_UI_PORT]) class SparkOnYarn(s.Service): JAR_FILE_TARGET = '/apps/spark/lib' MFS_DIR = '/apps/spark' SERVLET_JAR = 'javax.servlet-api.jar' def __init__(self): super(SparkOnYarn, self).__init__() self._name = 'spark'
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu HTTP_FS = np.NodeProcess( name='httpfs', ui_name='HTTPFS', package='mapr-httpfs', open_ports=[14000] ) class HttpFS(s.Service): def __init__(self): super(HttpFS, self).__init__() self._name = 'httpfs' self._ui_name = 'HttpFS' self._version = '1.0' self._node_processes = [HTTP_FS] self._cluster_defaults = ['httpfs-default.json'] self._validation_rules = [vu.exactly(1, HTTP_FS)] def post_install(self, cluster_context, instances):
# License for the specific language governing permissions and limitations # under the License. import six import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu from sahara.plugins.mapr.versions import version_handler_factory as vhf from sahara.swift import swift_helper from sahara.topology import topology_helper as topo from sahara.utils import files as f JOB_TRACKER = np.NodeProcess(name='jobtracker', ui_name='JobTracker', package='mapr-jobtracker', open_ports=[9001, 50030]) TASK_TRACKER = np.NodeProcess(name='tasktracker', ui_name='TaskTracker', package='mapr-tasktracker', open_ports=[50060]) JACKSON_CORE_ASL = ('plugins/mapr/services/swift/resources/' 'jackson-core-asl-1.9.13.jar') JACKSON_MAPPER_ASL = ('plugins/mapr/services/swift/resources/' 'jackson-mapper-asl-1.9.13.jar') @six.add_metaclass(s.Single) class MapReduce(s.Service): cluster_mode = 'classic'
# not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu PIG = np.NodeProcess(name='pig', ui_name='Pig', package='mapr-pig') class Pig(s.Service): def __init__(self): super(Pig, self).__init__() self._name = 'pig' self._ui_name = 'Pig' self._node_processes = [PIG] self._validation_rules = [vu.at_least(1, PIG)] class PigV013(Pig): def __init__(self): super(PigV013, self).__init__() self._version = '0.13'
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu MAHOUT = np.NodeProcess( name='mahout', ui_name='Mahout', package='mapr-mahout', open_ports=[] ) class Mahout(s.Service): def __init__(self): super(Mahout, self).__init__() self._name = 'mahout' self._ui_name = 'Mahout' self._node_processes = [MAHOUT] self._validation_rules = [vu.at_least(1, MAHOUT)] class MahoutV09(Mahout): def __init__(self):
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu HBASE_MASTER = np.NodeProcess( name='hbmaster', ui_name='HBase-Master', package='mapr-hbase-master', open_ports=[60000, 60010] ) HBASE_REGION_SERVER = np.NodeProcess( name='hbregionserver', ui_name='HBase-RegionServer', package='mapr-hbase-regionserver', open_ports=[60020] ) HBASE_THRIFT = np.NodeProcess( name='hbasethrift', ui_name='HBase-Thrift', package='mapr-hbasethrift', open_ports=[9090] ) HBASE_REST = np.NodeProcess(
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu KAFKA = np.NodeProcess(name='kafka', ui_name='Kafka', package='mapr-kafka', open_ports=[9092]) KAFKA_REST = np.NodeProcess(name='kafka', ui_name='Kafka Rest', package='mapr-kafka-rest', open_ports=[8082]) KAFKA_CONNECT_HDFS = np.NodeProcess(name='kafka', ui_name='Kafka Connect HDFS', package='mapr-kafka-connect-hdfs') KAFKA_CONNECT_JDBC = np.NodeProcess(name='kafka', ui_name='Kafka Connect JDBC', package='mapr-kafka-connect-jdbc')
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu FLUME = np.NodeProcess(name='flume', ui_name='Flume', package='mapr-flume', open_ports=[]) class Flume(s.Service): def __init__(self): super(Flume, self).__init__() self._name = 'flume' self._ui_name = 'Flume' self._node_processes = [FLUME] self._validation_rules = [vu.at_least(1, FLUME)] class FlumeV15(Flume): def __init__(self): super(FlumeV15, self).__init__()
from sahara import context from sahara.i18n import _ import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.event_log as el import sahara.plugins.mapr.util.general as g import sahara.plugins.mapr.util.validation_utils as vu import sahara.plugins.provisioning as p from sahara.utils import cluster_progress_ops as cpo from sahara.utils import files LOG = logging.getLogger(__name__) CLDB = np.NodeProcess(name='cldb', ui_name='CLDB', package='mapr-cldb', open_ports=[7222, 7220, 7221]) FILE_SERVER = np.NodeProcess(name='fileserver', ui_name='FileServer', package='mapr-fileserver', open_ports=[]) NFS = np.NodeProcess(name='nfs', ui_name='NFS', package='mapr-nfs', open_ports=[2049, 9997, 9998]) class MapRFS(s.Service): _CREATE_DISK_LIST = 'plugins/mapr/resources/create_disk_list_file.sh' _DISK_SETUP_CMD = '/opt/mapr/server/disksetup -F /tmp/disk.list' _DISK_SETUP_TIMEOUT = 600
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.commands as cmd import sahara.plugins.mapr.util.validation_utils as vu ZK_CLIENT_PORT = 5181 ZOOKEEPER = np.NodeProcess( name='mapr-zookeeper', ui_name='ZooKeeper', package='mapr-zookeeper', open_ports=[ZK_CLIENT_PORT] ) WEB_SERVER = np.NodeProcess( name='webserver', ui_name='Webserver', package='mapr-webserver', open_ports=[8443] ) METRICS = np.NodeProcess( name='metrics', ui_name='Metrics', package='mapr-metrics', open_ports=[1111]
# under the License. import six import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu from sahara.swift import swift_helper from sahara.topology import topology_helper as topo RESOURCE_MANAGER = np.NodeProcess( name='resourcemanager', ui_name='ResourceManager', package='mapr-resourcemanager', open_ports=[8033, 8032, 8031, 8030, 8088] ) NODE_MANAGER = np.NodeProcess( name='nodemanager', ui_name='NodeManager', package='mapr-nodemanager', open_ports=[8041, 8040, 8042, 8044] ) HISTORY_SERVER = np.NodeProcess( name='historyserver', ui_name='HistoryServer', package='mapr-historyserver', open_ports=[10020, 19888, 19890] )
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import six import sahara.plugins.mapr.domain.configuration_file as bcf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.util.validation_utils as vu HBASE_MASTER = np.NodeProcess(name='hbmaster', ui_name='HBase-Master', package='mapr-hbase-master', open_ports=[60000, 60010]) HBASE_REGION_SERVER = np.NodeProcess(name='hbregionserver', ui_name='HBase-RegionServer', package='mapr-hbase-regionserver', open_ports=[60020]) HBASE_THRIFT = np.NodeProcess(name='hbasethrift', ui_name='HBase-Thrift', package='mapr-hbasethrift', open_ports=[9090]) class HBase(s.Service): def __init__(self): super(HBase, self).__init__() self._name = 'hbase'
import sahara.plugins.mapr.services.oozie.oozie as oozie import sahara.plugins.mapr.services.sentry.sentry as sentry import sahara.plugins.mapr.services.spark.spark as spark import sahara.plugins.mapr.services.sqoop.sqoop2 as sqoop import sahara.plugins.mapr.services.yarn.yarn as yarn import sahara.plugins.mapr.util.event_log as el import sahara.plugins.mapr.util.general as g import sahara.plugins.mapr.util.password_utils as pu import sahara.plugins.mapr.util.validation_utils as vu import sahara.plugins.provisioning as p import sahara.utils.files as files LOG = logging.getLogger(__name__) HUE = np.NodeProcess(name='hue', ui_name='Hue', package='mapr-hue', open_ports=[8002, 8888]) HUE_LIVY = np.NodeProcess(name="livy", ui_name="Hue Livy", package="mapr-hue-livy", open_ports=[8998]) class Hue(s.Service): THRIFT_VERSIONS = [5, 7] THRIFT_VERSION = p.Config(name="Thrift version", applicable_target="Hue", scope='cluster', config_type="dropdown", config_values=[(v, v) for v in THRIFT_VERSIONS],
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from sahara.i18n import _ import sahara.plugins.mapr.domain.configuration_file as cf import sahara.plugins.mapr.domain.node_process as np import sahara.plugins.mapr.domain.service as s import sahara.plugins.mapr.services.mysql.mysql as mysql import sahara.plugins.mapr.util.maprfs_helper as mfs import sahara.plugins.provisioning as p import sahara.utils.files as files SENTRY = np.NodeProcess( name='sentry', ui_name='Sentry', package='mapr-sentry' ) SENTRY_MODE_CONFIG_NAME = 'Sentry storage mode' FILE_STORAGE_SENTRY_MODE = 'File-base storage' DB_STORAGE_SENTRY_MODE = 'DB-based storage' class Sentry(s.Service): SENTRY_STORAGE_MODE = p.Config( name=SENTRY_MODE_CONFIG_NAME, applicable_target='Sentry', scope='cluster', config_type="dropdown", config_values=[(v, v) for v in