def test_get_service_name_by_node_process(self):
ctx = self._get_context()
s_name_1 = ctx.get_service_name_by_node_process(yarn.RESOURCE_MANAGER)
self.assertEqual(yarn.YARN().ui_name, s_name_1)
s_name_2 = ctx.get_service_name_by_node_process(
yarn.RESOURCE_MANAGER.ui_name)
self.assertEqual(yarn.YARN().ui_name, s_name_2)
not_existing_np = np.NodeProcess('not_existing', 'NotExisting', 'foo')
self.assertIsNone(
ctx.get_service_name_by_node_process(not_existing_np))
self.assertIsNone(
ctx.get_service_name_by_node_process(not_existing_np.ui_name))
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
PIG = np.NodeProcess(name='pig', ui_name='Pig', package='mapr-pig')
class Pig(s.Service):
def __init__(self):
super(Pig, self).__init__()
self._name = 'pig'
self._ui_name = 'Pig'
self._node_processes = [PIG]
self._validation_rules = [vu.at_least(1, PIG)]
class PigV013(Pig):
def __init__(self):
super(PigV013, self).__init__()
self._version = '0.13'
import sahara_plugins.plugins.mapr.services.mapreduce.mapreduce as mr
import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql
import sahara_plugins.plugins.mapr.services.oozie.oozie as oozie
import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry
import sahara_plugins.plugins.mapr.services.spark.spark as spark
import sahara_plugins.plugins.mapr.services.sqoop.sqoop2 as sqoop
import sahara_plugins.plugins.mapr.services.yarn.yarn as yarn
import sahara_plugins.plugins.mapr.util.event_log as el
import sahara_plugins.plugins.mapr.util.general as g
import sahara_plugins.plugins.mapr.util.password_utils as pu
import sahara_plugins.plugins.mapr.util.validation_utils as vu
LOG = logging.getLogger(__name__)
HUE = np.NodeProcess(name='hue',
ui_name='Hue',
package='mapr-hue',
open_ports=[8002, 8888])
HUE_LIVY = np.NodeProcess(name="livy",
ui_name="Hue Livy",
package="mapr-hue-livy",
open_ports=[8998])
class Hue(s.Service):
THRIFT_VERSIONS = [5, 7]
THRIFT_VERSION = p.Config(name="Thrift version",
applicable_target="Hue",
scope='cluster',
config_type="dropdown",
config_values=[(v, v) for v in THRIFT_VERSIONS],
def __init__(self, *args, **kwds):
super(TestClusterContext, self).__init__(*args, **kwds)
self.fake_np = np.NodeProcess('fake', 'foo', 'bar')
from sahara.plugins import context
import sahara.plugins.provisioning as p
from sahara.plugins import utils
from sahara_plugins.i18n import _
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.event_log as el
import sahara_plugins.plugins.mapr.util.general as g
import sahara_plugins.plugins.mapr.util.validation_utils as vu
LOG = logging.getLogger(__name__)
CLDB = np.NodeProcess(
name='cldb',
ui_name='CLDB',
package='mapr-cldb',
open_ports=[7222, 7220, 7221]
)
FILE_SERVER = np.NodeProcess(
name='fileserver',
ui_name='FileServer',
package='mapr-fileserver',
open_ports=[]
)
NFS = np.NodeProcess(
name='nfs',
ui_name='NFS',
package='mapr-nfs',
open_ports=[2049, 9997, 9998]
)
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.commands as cmd
import sahara_plugins.plugins.mapr.util.validation_utils as vu
DRILL = np.NodeProcess(name='drill-bits',
ui_name='Drill',
package='mapr-drill',
open_ports=[8047])
DRILL_YARN = np.NodeProcess(name='drill-bits-yarn',
ui_name='Drill on YARN',
package='mapr-drill-yarn',
open_ports=[8047])
class Drill(s.Service):
def __init__(self):
super(Drill, self).__init__()
self._name = 'drill'
self._ui_name = 'Drill'
self._node_processes = [DRILL]
self._ui_info = [('Drill', DRILL, {s.SERVICE_UI: 'http://%s:8047'})]
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
FLUME = np.NodeProcess(name='flume',
ui_name='Flume',
package='mapr-flume',
open_ports=[])
class Flume(s.Service):
def __init__(self):
super(Flume, self).__init__()
self._name = 'flume'
self._ui_name = 'Flume'
self._node_processes = [FLUME]
self._validation_rules = [vu.at_least(1, FLUME)]
class FlumeV15(Flume):
def __init__(self):
super(FlumeV15, self).__init__()
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
HTTP_FS = np.NodeProcess(name='httpfs',
ui_name='HTTPFS',
package='mapr-httpfs',
open_ports=[14000])
class HttpFS(s.Service):
def __init__(self):
super(HttpFS, self).__init__()
self._name = 'httpfs'
self._ui_name = 'HttpFS'
self._version = '1.0'
self._node_processes = [HTTP_FS]
self._cluster_defaults = ['httpfs-default.json']
self._validation_rules = [vu.exactly(1, HTTP_FS)]
def post_install(self, cluster_context, instances):
self._set_service_dir_owner(cluster_context, instances)
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.utils as utils
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.services.hbase.hbase as hbase
import sahara_plugins.plugins.mapr.services.hive.hive as hive
import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry
import sahara_plugins.plugins.mapr.util.general as g
import sahara_plugins.plugins.mapr.util.validation_utils as vu
IMPALA_SERVER = np.NodeProcess(name='impalaserver',
ui_name='Impala-Server',
package='mapr-impala-server',
open_ports=[21000, 21050, 25000])
IMPALA_STATE_STORE = np.NodeProcess(name='impalastore',
ui_name='Impala-Statestore',
package='mapr-impala-statestore',
open_ports=[25010])
IMPALA_CATALOG = np.NodeProcess(name='impalacatalog',
ui_name='Impala-Catalog',
package='mapr-impala-catalog',
open_ports=[25020])
class Impala(s.Service):
def __init__(self):
super(Impala, self).__init__()
self._name = 'impala'
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.password_utils as pu
import sahara_plugins.plugins.mapr.util.validation_utils as vu
ZK_CLIENT_PORT = 5181
ZOOKEEPER = np.NodeProcess(name='mapr-zookeeper',
ui_name='ZooKeeper',
package='mapr-zookeeper',
open_ports=[ZK_CLIENT_PORT])
WEB_SERVER = np.NodeProcess(name='webserver',
ui_name='Webserver',
package='mapr-webserver',
open_ports=[8443])
METRICS = np.NodeProcess(name='metrics',
ui_name='Metrics',
package='mapr-metrics',
open_ports=[1111])
class Management(s.Service):
def __init__(self):
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sahara.plugins import swift_helper
from sahara.plugins import topology_helper as topo
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
JOB_TRACKER = np.NodeProcess(name='jobtracker',
ui_name='JobTracker',
package='mapr-jobtracker',
open_ports=[9001, 50030])
TASK_TRACKER = np.NodeProcess(name='tasktracker',
ui_name='TaskTracker',
package='mapr-tasktracker',
open_ports=[50060])
class MapReduce(s.Service):
cluster_mode = 'classic'
def __init__(self):
super(MapReduce, self).__init__()
self._ui_name = 'MapReduce'
self._name = 'hadoop'
self._version = '0.20.2'
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
HBASE_MASTER = np.NodeProcess(
name='hbmaster',
ui_name='HBase-Master',
package='mapr-hbase-master',
open_ports=[60000, 60010]
)
HBASE_REGION_SERVER = np.NodeProcess(
name='hbregionserver',
ui_name='HBase-RegionServer',
package='mapr-hbase-regionserver',
open_ports=[60020]
)
HBASE_THRIFT = np.NodeProcess(
name='hbasethrift',
ui_name='HBase-Thrift',
package='mapr-hbasethrift',
open_ports=[9090]
)
HBASE_REST = np.NodeProcess(
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
KAFKA = np.NodeProcess(name='kafka',
ui_name='Kafka',
package='mapr-kafka',
open_ports=[9092])
KAFKA_REST = np.NodeProcess(name='kafka',
ui_name='Kafka Rest',
package='mapr-kafka-rest',
open_ports=[8082])
KAFKA_CONNECT_HDFS = np.NodeProcess(name='kafka',
ui_name='Kafka Connect HDFS',
package='mapr-kafka-connect-hdfs')
KAFKA_CONNECT_JDBC = np.NodeProcess(name='kafka',
ui_name='Kafka Connect JDBC',
package='mapr-kafka-connect-jdbc')
import sahara.plugins.context as context
from sahara_plugins.i18n import _
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql
import sahara_plugins.plugins.mapr.util.event_log as el
import sahara_plugins.plugins.mapr.util.general as g
import sahara_plugins.plugins.mapr.util.validation_utils as vu
LOG = logging.getLogger(__name__)
OOZIE_START_DELAY = 30
OOZIE = np.NodeProcess(
name='oozie',
ui_name='Oozie',
package='mapr-oozie',
open_ports=[11000]
)
class Oozie(s.Service):
def __init__(self):
super(Oozie, self).__init__()
self._name = 'oozie'
self._ui_name = 'Oozie'
self._node_processes = [OOZIE]
self._cluster_defaults = ['oozie-default.json']
self._validation_rules = [vu.exactly(1, OOZIE)]
self._ui_info = [('Oozie', OOZIE,
{s.SERVICE_UI: 'http://%s:11000/oozie'})]
import sahara.plugins.utils as utils
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.services.hbase.hbase as hbase
import sahara_plugins.plugins.mapr.services.hive.hive as hive
import sahara_plugins.plugins.mapr.util.general as g
import sahara_plugins.plugins.mapr.util.maprfs_helper as mfs
import sahara_plugins.plugins.mapr.util.validation_utils as vu
SPARK_SLAVE_UI_PORT = 8081
SPARK_HS_UI_PORT = 18080
SPARK_HISTORY_SERVER = np.NodeProcess(name='spark-historyserver',
ui_name='Spark HistoryServer',
package='mapr-spark-historyserver',
open_ports=[SPARK_HS_UI_PORT])
SPARK_SLAVE = np.NodeProcess(name='spark-master',
ui_name='Spark Slave',
package='mapr-spark',
open_ports=[SPARK_SLAVE_UI_PORT])
class SparkOnYarn(s.Service):
JAR_FILE_TARGET = '/apps/spark/lib'
MFS_DIR = '/apps/spark'
SERVLET_JAR = 'javax.servlet-api.jar'
def __init__(self):
super(SparkOnYarn, self).__init__()
self._name = 'spark'
from oslo_log import log as logging
from sahara.i18n import _
import sahara_plugins.plugins.mapr.domain.configuration_file as bcf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.services.sentry.sentry as sentry
import sahara_plugins.plugins.mapr.util.event_log as el
import sahara_plugins.plugins.mapr.util.validation_utils as vu
import sahara.utils.files as files
LOG = logging.getLogger(__name__)
HIVE_METASTORE = np.NodeProcess(name='hivemeta',
ui_name='HiveMetastore',
package='mapr-hivemetastore',
open_ports=[9083])
HIVE_SERVER_2 = np.NodeProcess(name='hs2',
ui_name='HiveServer2',
package='mapr-hiveserver2',
open_ports=[10000])
class Hive(s.Service):
def __init__(self):
super(Hive, self).__init__()
self._name = 'hive'
self._ui_name = 'Hive'
self._node_processes = [HIVE_METASTORE, HIVE_SERVER_2]
self._validation_rules = [
vu.exactly(1, HIVE_METASTORE),
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
SQOOP_2_SERVER = np.NodeProcess(name='sqoop2',
ui_name='Sqoop2-Server',
package='mapr-sqoop2-server',
open_ports=[12000])
SQOOP_2_CLIENT = np.NodeProcess(name='sqoop-client',
ui_name='Sqoop2-Client',
package='mapr-sqoop2-client')
class Sqoop2(s.Service):
def __init__(self):
super(Sqoop2, self).__init__()
self._name = 'sqoop'
self._ui_name = 'Sqoop2'
self._version = '2.0.0'
self._node_processes = [SQOOP_2_CLIENT, SQOOP_2_SERVER]
self._validation_rules = [
vu.exactly(1, SQOOP_2_SERVER),
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.util.validation_utils as vu
MAHOUT = np.NodeProcess(name='mahout',
ui_name='Mahout',
package='mapr-mahout',
open_ports=[])
class Mahout(s.Service):
def __init__(self):
super(Mahout, self).__init__()
self._name = 'mahout'
self._ui_name = 'Mahout'
self._node_processes = [MAHOUT]
self._validation_rules = [vu.at_least(1, MAHOUT)]
class MahoutV09(Mahout):
def __init__(self):
super(MahoutV09, self).__init__()
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sahara.plugins.provisioning as p
import sahara.plugins.utils as utils
from sahara_plugins.i18n import _
import sahara_plugins.plugins.mapr.domain.configuration_file as cf
import sahara_plugins.plugins.mapr.domain.node_process as np
import sahara_plugins.plugins.mapr.domain.service as s
import sahara_plugins.plugins.mapr.services.mysql.mysql as mysql
import sahara_plugins.plugins.mapr.util.event_log as el
import sahara_plugins.plugins.mapr.util.maprfs_helper as mfs
SENTRY = np.NodeProcess(name='sentry', ui_name='Sentry', package='mapr-sentry')
SENTRY_MODE_CONFIG_NAME = 'Sentry storage mode'
FILE_STORAGE_SENTRY_MODE = 'File-base storage'
DB_STORAGE_SENTRY_MODE = 'DB-based storage'
class Sentry(s.Service):
SENTRY_STORAGE_MODE = p.Config(
name=SENTRY_MODE_CONFIG_NAME,
applicable_target='Sentry',
scope='cluster',
config_type="dropdown",
config_values=[
(v, v) for v in (FILE_STORAGE_SENTRY_MODE, DB_STORAGE_SENTRY_MODE)
],