def test_get_service_name_by_node_process(self):
ctx = self._get_context()
s_name_1 = ctx.get_service_name_by_node_process(yarn.RESOURCE_MANAGER)
self.assertEqual(yarn.YARN().ui_name, s_name_1)
s_name_2 = ctx.get_service_name_by_node_process(
yarn.RESOURCE_MANAGER.ui_name)
self.assertEqual(yarn.YARN().ui_name, s_name_2)
not_existing_np = np.NodeProcess('not_existing', 'NotExisting', 'foo')
self.assertIsNone(
ctx.get_service_name_by_node_process(not_existing_np))
self.assertIsNone(
ctx.get_service_name_by_node_process(not_existing_np.ui_name))
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
import six
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.utils.files as files
LOG = logging.getLogger(__name__)
HIVE_METASTORE = np.NodeProcess(name='hivemeta',
ui_name='HiveMetastore',
package='mapr-hivemetastore',
open_ports=[9083])
HIVE_SERVER_2 = np.NodeProcess(name='hs2',
ui_name='HiveServer2',
package='mapr-hiveserver2',
open_ports=[10000])
class Hive(s.Service):
def __init__(self):
super(Hive, self).__init__()
self._name = 'hive'
self._ui_name = 'Hive'
self._node_processes = [HIVE_METASTORE, HIVE_SERVER_2]
self._validation_rules = [
vu.exactly(1, HIVE_METASTORE),
import sahara.context as context
from sahara.i18n import _
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.services.mysql.mysql as mysql
import sahara.plugins.mapr.util.event_log as el
import sahara.plugins.mapr.util.general as g
import sahara.plugins.mapr.util.validation_utils as vu
LOG = logging.getLogger(__name__)
OOZIE_START_DELAY = 30
OOZIE = np.NodeProcess(name='oozie',
ui_name='Oozie',
package='mapr-oozie',
open_ports=[11000])
class Oozie(s.Service):
def __init__(self):
super(Oozie, self).__init__()
self._name = 'oozie'
self._ui_name = 'Oozie'
self._node_processes = [OOZIE]
self._cluster_defaults = ['oozie-default.json']
self._validation_rules = [vu.exactly(1, OOZIE)]
self._ui_info = [('Oozie', OOZIE, {
s.SERVICE_UI: 'http://%s:11000/oozie'
})]
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.services.hive.hive as hive
import sahara.plugins.mapr.util.commands as cmd
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.utils.files as files
IMPALA_SERVER = np.NodeProcess(name='impalaserver',
ui_name='Impala-Server',
package='mapr-impala-server',
open_ports=[21000, 21050, 25000])
IMPALA_STATE_STORE = np.NodeProcess(name='impalastore',
ui_name='Impala-Statestore',
package='mapr-impala-statestore',
open_ports=[25010])
IMPALA_CATALOG = np.NodeProcess(name='impalacatalog',
ui_name='Impala-Catalog',
package='mapr-impala-catalog',
open_ports=[25020])
class Impala(s.Service):
def __init__(self):
super(Impala, self).__init__()
self._name = 'impala'
import sahara.plugins.mapr.services.impala.impala as impala
import sahara.plugins.mapr.services.mapreduce.mapreduce as mr
import sahara.plugins.mapr.services.mysql.mysql as mysql
import sahara.plugins.mapr.services.oozie.oozie as oozie
import sahara.plugins.mapr.services.sqoop.sqoop2 as sqoop
import sahara.plugins.mapr.services.yarn.yarn as yarn
import sahara.plugins.mapr.util.general as g
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.utils.files as files
LOG = logging.getLogger(__name__)
HUE = np.NodeProcess(
name='hue',
ui_name='Hue',
package='mapr-hue',
open_ports=[8002, 8888]
)
@six.add_metaclass(s.Single)
class Hue(s.Service):
def __init__(self):
super(Hue, self).__init__()
self._name = 'hue'
self._ui_name = 'Hue'
self._version = '3.6.0'
self._node_processes = [HUE]
self._ui_info = [('HUE', HUE, 'http://%s:8888')]
self._validation_rules = [
vu.exactly(1, HUE),
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.commands as cmd
import sahara.plugins.mapr.util.validation_utils as vu
SQOOP_2_SERVER = np.NodeProcess(name='sqoop2',
ui_name='Sqoop2-Server',
package='mapr-sqoop2-server',
open_ports=[12000])
SQOOP_2_CLIENT = np.NodeProcess(name='sqoop-client',
ui_name='Sqoop2-Client',
package='mapr-sqoop2-client')
@six.add_metaclass(s.Single)
class Sqoop2(s.Service):
def __init__(self):
super(Sqoop2, self).__init__()
self._name = 'sqoop'
self._ui_name = 'Sqoop2'
self._version = '2.0.0'
self._node_processes = [SQOOP_2_CLIENT, SQOOP_2_SERVER]
self._validation_rules = [
def __init__(self, *args, **kwds):
super(TestClusterContext, self).__init__(*args, **kwds)
self.fake_np = np.NodeProcess('fake', 'foo', 'bar')
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.commands as cmd
import sahara.plugins.mapr.util.validation_utils as vu
DRILL = np.NodeProcess(
name='drill-bits',
ui_name='Drill',
package='mapr-drill',
open_ports=[]
)
class Drill(s.Service):
def __init__(self):
super(Drill, self).__init__()
self._name = 'drill'
self._ui_name = 'Drill'
self._node_processes = [DRILL]
self._ui_info = [('Drill', DRILL, 'http://%s:8047')]
self._validation_rules = [vu.at_least(1, DRILL)]
def install(self, cluster_context, instances):
# Drill requires running cluster
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.services.hbase.hbase as hbase
import sahara.plugins.mapr.services.hive.hive as hive
import sahara.plugins.mapr.util.general as g
import sahara.plugins.mapr.util.maprfs_helper as mfs
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.utils.files as files
SPARK_SLAVE_UI_PORT = 8081
SPARK_HS_UI_PORT = 18080
SPARK_HISTORY_SERVER = np.NodeProcess(name='spark-historyserver',
ui_name='Spark HistoryServer',
package='mapr-spark-historyserver',
open_ports=[SPARK_HS_UI_PORT])
SPARK_SLAVE = np.NodeProcess(name='spark-master',
ui_name='Spark Slave',
package='mapr-spark',
open_ports=[SPARK_SLAVE_UI_PORT])
class SparkOnYarn(s.Service):
JAR_FILE_TARGET = '/apps/spark/lib'
MFS_DIR = '/apps/spark'
SERVLET_JAR = 'javax.servlet-api.jar'
def __init__(self):
super(SparkOnYarn, self).__init__()
self._name = 'spark'
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
HTTP_FS = np.NodeProcess(
name='httpfs',
ui_name='HTTPFS',
package='mapr-httpfs',
open_ports=[14000]
)
class HttpFS(s.Service):
def __init__(self):
super(HttpFS, self).__init__()
self._name = 'httpfs'
self._ui_name = 'HttpFS'
self._version = '1.0'
self._node_processes = [HTTP_FS]
self._cluster_defaults = ['httpfs-default.json']
self._validation_rules = [vu.exactly(1, HTTP_FS)]
def post_install(self, cluster_context, instances):
# License for the specific language governing permissions and limitations
# under the License.
import six
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
from sahara.plugins.mapr.versions import version_handler_factory as vhf
from sahara.swift import swift_helper
from sahara.topology import topology_helper as topo
from sahara.utils import files as f
JOB_TRACKER = np.NodeProcess(name='jobtracker',
ui_name='JobTracker',
package='mapr-jobtracker',
open_ports=[9001, 50030])
TASK_TRACKER = np.NodeProcess(name='tasktracker',
ui_name='TaskTracker',
package='mapr-tasktracker',
open_ports=[50060])
JACKSON_CORE_ASL = ('plugins/mapr/services/swift/resources/'
'jackson-core-asl-1.9.13.jar')
JACKSON_MAPPER_ASL = ('plugins/mapr/services/swift/resources/'
'jackson-mapper-asl-1.9.13.jar')
@six.add_metaclass(s.Single)
class MapReduce(s.Service):
cluster_mode = 'classic'
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
PIG = np.NodeProcess(name='pig', ui_name='Pig', package='mapr-pig')
class Pig(s.Service):
def __init__(self):
super(Pig, self).__init__()
self._name = 'pig'
self._ui_name = 'Pig'
self._node_processes = [PIG]
self._validation_rules = [vu.at_least(1, PIG)]
class PigV013(Pig):
def __init__(self):
super(PigV013, self).__init__()
self._version = '0.13'
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
MAHOUT = np.NodeProcess(
name='mahout',
ui_name='Mahout',
package='mapr-mahout',
open_ports=[]
)
class Mahout(s.Service):
def __init__(self):
super(Mahout, self).__init__()
self._name = 'mahout'
self._ui_name = 'Mahout'
self._node_processes = [MAHOUT]
self._validation_rules = [vu.at_least(1, MAHOUT)]
class MahoutV09(Mahout):
def __init__(self):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
HBASE_MASTER = np.NodeProcess(
name='hbmaster',
ui_name='HBase-Master',
package='mapr-hbase-master',
open_ports=[60000, 60010]
)
HBASE_REGION_SERVER = np.NodeProcess(
name='hbregionserver',
ui_name='HBase-RegionServer',
package='mapr-hbase-regionserver',
open_ports=[60020]
)
HBASE_THRIFT = np.NodeProcess(
name='hbasethrift',
ui_name='HBase-Thrift',
package='mapr-hbasethrift',
open_ports=[9090]
)
HBASE_REST = np.NodeProcess(
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
KAFKA = np.NodeProcess(name='kafka',
ui_name='Kafka',
package='mapr-kafka',
open_ports=[9092])
KAFKA_REST = np.NodeProcess(name='kafka',
ui_name='Kafka Rest',
package='mapr-kafka-rest',
open_ports=[8082])
KAFKA_CONNECT_HDFS = np.NodeProcess(name='kafka',
ui_name='Kafka Connect HDFS',
package='mapr-kafka-connect-hdfs')
KAFKA_CONNECT_JDBC = np.NodeProcess(name='kafka',
ui_name='Kafka Connect JDBC',
package='mapr-kafka-connect-jdbc')
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
FLUME = np.NodeProcess(name='flume',
ui_name='Flume',
package='mapr-flume',
open_ports=[])
class Flume(s.Service):
def __init__(self):
super(Flume, self).__init__()
self._name = 'flume'
self._ui_name = 'Flume'
self._node_processes = [FLUME]
self._validation_rules = [vu.at_least(1, FLUME)]
class FlumeV15(Flume):
def __init__(self):
super(FlumeV15, self).__init__()
from sahara import context
from sahara.i18n import _
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.event_log as el
import sahara.plugins.mapr.util.general as g
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.plugins.provisioning as p
from sahara.utils import cluster_progress_ops as cpo
from sahara.utils import files
LOG = logging.getLogger(__name__)
CLDB = np.NodeProcess(name='cldb',
ui_name='CLDB',
package='mapr-cldb',
open_ports=[7222, 7220, 7221])
FILE_SERVER = np.NodeProcess(name='fileserver',
ui_name='FileServer',
package='mapr-fileserver',
open_ports=[])
NFS = np.NodeProcess(name='nfs',
ui_name='NFS',
package='mapr-nfs',
open_ports=[2049, 9997, 9998])
class MapRFS(s.Service):
_CREATE_DISK_LIST = 'plugins/mapr/resources/create_disk_list_file.sh'
_DISK_SETUP_CMD = '/opt/mapr/server/disksetup -F /tmp/disk.list'
_DISK_SETUP_TIMEOUT = 600
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.commands as cmd
import sahara.plugins.mapr.util.validation_utils as vu
ZK_CLIENT_PORT = 5181
ZOOKEEPER = np.NodeProcess(
name='mapr-zookeeper',
ui_name='ZooKeeper',
package='mapr-zookeeper',
open_ports=[ZK_CLIENT_PORT]
)
WEB_SERVER = np.NodeProcess(
name='webserver',
ui_name='Webserver',
package='mapr-webserver',
open_ports=[8443]
)
METRICS = np.NodeProcess(
name='metrics',
ui_name='Metrics',
package='mapr-metrics',
open_ports=[1111]
# under the License.
import six
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
from sahara.swift import swift_helper
from sahara.topology import topology_helper as topo
RESOURCE_MANAGER = np.NodeProcess(
name='resourcemanager',
ui_name='ResourceManager',
package='mapr-resourcemanager',
open_ports=[8033, 8032, 8031, 8030, 8088]
)
NODE_MANAGER = np.NodeProcess(
name='nodemanager',
ui_name='NodeManager',
package='mapr-nodemanager',
open_ports=[8041, 8040, 8042, 8044]
)
HISTORY_SERVER = np.NodeProcess(
name='historyserver',
ui_name='HistoryServer',
package='mapr-historyserver',
open_ports=[10020, 19888, 19890]
)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import sahara.plugins.mapr.domain.configuration_file as bcf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.util.validation_utils as vu
HBASE_MASTER = np.NodeProcess(name='hbmaster',
ui_name='HBase-Master',
package='mapr-hbase-master',
open_ports=[60000, 60010])
HBASE_REGION_SERVER = np.NodeProcess(name='hbregionserver',
ui_name='HBase-RegionServer',
package='mapr-hbase-regionserver',
open_ports=[60020])
HBASE_THRIFT = np.NodeProcess(name='hbasethrift',
ui_name='HBase-Thrift',
package='mapr-hbasethrift',
open_ports=[9090])
class HBase(s.Service):
def __init__(self):
super(HBase, self).__init__()
self._name = 'hbase'
import sahara.plugins.mapr.services.oozie.oozie as oozie
import sahara.plugins.mapr.services.sentry.sentry as sentry
import sahara.plugins.mapr.services.spark.spark as spark
import sahara.plugins.mapr.services.sqoop.sqoop2 as sqoop
import sahara.plugins.mapr.services.yarn.yarn as yarn
import sahara.plugins.mapr.util.event_log as el
import sahara.plugins.mapr.util.general as g
import sahara.plugins.mapr.util.password_utils as pu
import sahara.plugins.mapr.util.validation_utils as vu
import sahara.plugins.provisioning as p
import sahara.utils.files as files
LOG = logging.getLogger(__name__)
HUE = np.NodeProcess(name='hue',
ui_name='Hue',
package='mapr-hue',
open_ports=[8002, 8888])
HUE_LIVY = np.NodeProcess(name="livy",
ui_name="Hue Livy",
package="mapr-hue-livy",
open_ports=[8998])
class Hue(s.Service):
THRIFT_VERSIONS = [5, 7]
THRIFT_VERSION = p.Config(name="Thrift version",
applicable_target="Hue",
scope='cluster',
config_type="dropdown",
config_values=[(v, v) for v in THRIFT_VERSIONS],
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sahara.i18n import _
import sahara.plugins.mapr.domain.configuration_file as cf
import sahara.plugins.mapr.domain.node_process as np
import sahara.plugins.mapr.domain.service as s
import sahara.plugins.mapr.services.mysql.mysql as mysql
import sahara.plugins.mapr.util.maprfs_helper as mfs
import sahara.plugins.provisioning as p
import sahara.utils.files as files
SENTRY = np.NodeProcess(
name='sentry',
ui_name='Sentry',
package='mapr-sentry'
)
SENTRY_MODE_CONFIG_NAME = 'Sentry storage mode'
FILE_STORAGE_SENTRY_MODE = 'File-base storage'
DB_STORAGE_SENTRY_MODE = 'DB-based storage'
class Sentry(s.Service):
SENTRY_STORAGE_MODE = p.Config(
name=SENTRY_MODE_CONFIG_NAME,
applicable_target='Sentry',
scope='cluster',
config_type="dropdown",
config_values=[(v, v) for v in
