def _create_config_obj(self,
item,
target='general',
scope='cluster',
high_priority=False):
def _prepare_value(value):
if isinstance(value, str):
return value.strip().lower()
return value
conf_name = _prepare_value(item.get('name', None))
conf_value = _prepare_value(item.get('value', None))
if not conf_name:
raise ex.HadoopProvisionError(_("Config missing 'name'"))
if conf_value is None:
raise ex.PluginInvalidDataException(
_("Config '%s' missing 'value'") % conf_name)
if high_priority or item.get('priority', 2) == 1:
priority = 1
else:
priority = 2
return p.Config(name=conf_name,
applicable_target=target,
scope=scope,
config_type=item.get('config_type', "string"),
config_values=item.get('config_values', None),
default_value=conf_value,
is_optional=item.get('is_optional', True),
description=item.get('description', None),
priority=priority)
def _await_cldb(self, cluster_context, instances=None, timeout=600):
instances = instances or cluster_context.get_instances()
cldb_node = cluster_context.get_instance(mfs.CLDB)
start_time = timeutils.utcnow()
retry_count = 0
with cldb_node.remote() as r:
LOG.debug("Waiting {count} seconds for CLDB initialization".format(
count=timeout))
while timeutils.delta_seconds(start_time,
timeutils.utcnow()) < timeout:
ec, out = r.execute_command(NODE_LIST_CMD,
raise_when_error=False)
resp = json.loads(out)
status = resp['status']
if str(status).lower() == 'ok':
ips = [n['ip'] for n in resp['data']]
retry_count += 1
for i in instances:
if (i.internal_ip not in ips
and (retry_count > DEFAULT_RETRY_COUNT)):
msg = _("Node failed to connect to CLDB: %s"
) % i.internal_ip
raise ex.HadoopProvisionError(msg)
break
else:
context.sleep(DELAY)
else:
raise ex.HadoopProvisionError(_("CLDB failed to start"))
def stop(self, cluster_context, instances=None):
instances = instances or cluster_context.get_instances()
zookeepers = cluster_context.filter_instances(instances, mng.ZOOKEEPER)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Stop ZooKeepers nodes"),
len(zookeepers))
self._stop_zk_nodes(zookeepers)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Stop Warden nodes"), len(instances))
self._stop_warden_on_nodes(instances)
def get_service(self, node_process):
ui_name = self.get_service_name_by_node_process(node_process)
if ui_name is None:
raise e.PluginInvalidDataException(
_('Service not found in services list'))
version = self.get_chosen_service_version(ui_name)
service = self._find_service_instance(ui_name, version)
if service is None:
raise e.PluginInvalidDataException(_('Can not map service'))
return service
class NodeRequiredServiceMissingException(e.RequiredServiceMissingException):
MISSING_MSG = _('Node "%(ng_name)s" is missing component %(component)s')
REQUIRED_MSG = _('%(message)s, required by %(required_by)s')
def __init__(self, service_name, ng_name, required_by=None):
super(NodeRequiredServiceMissingException,
self).__init__(service_name, required_by)
args = {'ng_name': ng_name, 'component': service_name}
self.message = (NodeRequiredServiceMissingException.MISSING_MSG % args)
if required_by:
args = {'message': self.message, 'required_by': required_by}
self.message = (NodeRequiredServiceMissingException.REQUIRED_MSG %
args)
def check_health(self):
instances = self.cluster_context.get_instances(
node_process=management.ZOOKEEPER)
active_count = 0
for instance in instances:
if self._is_zookeeper_running(instance):
active_count += 1
if active_count == 0:
raise health_check_base.RedHealthError(
_("ZooKeeper is not in running state"))
if active_count < len(instances):
raise health_check_base.YellowHealthError(
_("Some ZooKeeper processes are not in running state"))
return _("ZooKeeper is in running state")
class NoVolumesException(e.SaharaPluginException):
MESSAGE = _('%s must have at least 1 volume or ephemeral drive')
ERROR_CODE = "NO_VOLUMES"
def __init__(self, ng_name):
super(NoVolumesException, self).__init__()
self.message = NoVolumesException.MESSAGE % ng_name
self.code = NoVolumesException.ERROR_CODE
class EvenCountException(e.SaharaPluginException):
MESSAGE = _("Hadoop cluster should contain odd number of %(component)s"
" but %(actual_count)s found.")
def __init__(self, component, count):
super(EvenCountException, self).__init__()
args = {'component': component, 'actual_count': count}
self.message = EvenCountException.MESSAGE % args
def get_node_process_name(node_process):
# This import is placed here to avoid circular imports
from sahara_plugin_mapr.plugins.mapr.domain import node_process as np # noqa
if isinstance(node_process, np.NodeProcess):
return node_process.ui_name
if isinstance(node_process, six.string_types):
return node_process
raise TypeError(_("Invalid argument type %s") % type(node_process))
class NotRequiredImageException(e.SaharaPluginException):
MESSAGE = _('Service %(service)s requires %(os)s OS.'
' Use %(os)s image and add "%(os)s" tag to it.')
ERROR_CODE = "INVALID_IMAGE"
def __init__(self, service, os):
super(NotRequiredImageException, self).__init__()
self.message = NotRequiredImageException.MESSAGE % {'service': service,
'os': os}
self.code = NotRequiredImageException.ERROR_CODE
def get_version_config(self, versions):
return p.Config(name='%s Version' % self._ui_name,
applicable_target=self.ui_name,
scope='cluster',
config_type='dropdown',
config_values=[
(v, v) for v in sorted(versions, reverse=True)
],
is_optional=False,
description=_('Specify the version of the service'),
priority=1)
def start(self, cluster_context, instances=None):
instances = instances or cluster_context.get_instances()
zookeepers = cluster_context.filter_instances(instances, mng.ZOOKEEPER)
cldbs = cluster_context.filter_instances(instances, mfs.CLDB)
others = filter(
lambda i: not cluster_context.check_for_process(i, mfs.CLDB),
instances)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Start ZooKeepers nodes"),
len(zookeepers))
self._start_zk_nodes(zookeepers)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Start CLDB nodes"), len(cldbs))
self._start_cldb_nodes(cldbs)
if others:
utils.add_provisioning_step(cluster_context.cluster.id,
_("Start non-CLDB nodes"),
len(list(others)))
self._start_non_cldb_nodes(others)
self._await_cldb(cluster_context, instances)
def post_install(self, cluster_context, instances):
LOG.debug('Initializing MapR FS')
instances = instances or cluster_context.get_instances()
file_servers = cluster_context.filter_instances(instances, FILE_SERVER)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Initializing MapR-FS"),
len(file_servers))
with context.PluginsThreadGroup() as tg:
for instance in file_servers:
tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance,
instance)
LOG.info('MapR FS successfully initialized')
class NodeServiceConflictException(e.SaharaPluginException):
MESSAGE = _('%(service)s service cannot be installed alongside'
' %(package)s package')
ERROR_CODE = "NODE_PROCESS_CONFLICT"
def __init__(self, service_name, conflicted_package):
super(NodeServiceConflictException, self).__init__()
args = {
'service': service_name,
'package': conflicted_package,
}
self.message = NodeServiceConflictException.MESSAGE % args
self.code = NodeServiceConflictException.ERROR_CODE
def _init_db_schema(self, cluster_context):
instance = cluster_context.get_instance(SENTRY)
cmd = '%(home)s/bin/sentry --command schema-tool' \
' --conffile %(home)s/conf/sentry-site.xml' \
' --dbType mysql --initSchema' % {
'home': self.home_dir(cluster_context)}
@el.provision_event(name=_("Init Sentry DB schema"), instance=instance)
def decorated():
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
decorated()
def _get_repo_configs(self):
ubuntu_base = p.Config(
name="Ubuntu base repo",
applicable_target="general",
scope='cluster',
priority=1,
default_value="",
description=_(
'Specifies Ubuntu MapR core repository.')
)
centos_base = p.Config(
name="CentOS base repo",
applicable_target="general",
scope='cluster',
priority=1,
default_value="",
description=_(
'Specifies CentOS MapR core repository.')
)
ubuntu_eco = p.Config(
name="Ubuntu ecosystem repo",
applicable_target="general",
scope='cluster',
priority=1,
default_value="",
description=_(
'Specifies Ubuntu MapR ecosystem repository.')
)
centos_eco = p.Config(
name="CentOS ecosystem repo",
applicable_target="general",
scope='cluster',
priority=1,
default_value="",
description=_(
'Specifies CentOS MapR ecosystem repository.')
)
return [ubuntu_base, centos_base, ubuntu_eco, centos_eco]
class LessThanCountException(e.InvalidComponentCountException):
MESSAGE = _("Hadoop cluster should contain at least"
" %(expected_count)d %(component)s component(s)."
" Actual %(component)s count is %(actual_count)d")
def __init__(self, component, expected_count, count):
super(LessThanCountException, self).__init__(
component, expected_count, count)
args = {
'expected_count': expected_count,
'component': component,
'actual_count': count,
}
self.message = LessThanCountException.MESSAGE % args
def install(self, cluster_context, instances):
service_instances = cluster_context.filter_instances(instances,
service=self)
@el.provision_step(_("Install %s service") % self.ui_name,
cluster_context_reference=0,
instances_reference=1)
def _install(_context, _instances):
g.execute_on_instances(_instances,
self._install_packages_on_instance,
_context)
if service_instances:
_install(cluster_context, service_instances)
def poll_status(instance):
operation_name = _('Wait for {node_process} on {instance}'
' to change status to "{status}"')
args = {
'node_process': self.ui_name,
'instance': instance.instance_name,
'status': status.name,
}
return plugin_utils.poll(
get_status=lambda: self.status(instance) == status,
operation_name=operation_name.format(**args),
timeout=timeout,
sleep=sleep,
)
def _configure_database(self, cluster_context, instances):
mysql_instance = mysql.MySQL.get_db_instance(cluster_context)
@el.provision_event(instance=mysql_instance,
name=_("Configure database"))
def decorated():
distro_name = cluster_context.distro.name
distro_version = cluster_context.distro_version
mysql.MySQL.install_mysql(mysql_instance, distro_name,
distro_version)
mysql.MySQL.start_mysql_server(cluster_context)
mysql.MySQL.create_databases(cluster_context, instances)
decorated()
class MoreThanCountException(e.InvalidComponentCountException):
MESSAGE = _("Hadoop cluster should contain at most"
" %(expected_count)d %(component)s component(s)."
" Actual %(component)s count is %(actual_count)d")
def __init__(self, component, expected_count, count):
super(MoreThanCountException, self).__init__(
component, expected_count, count)
args = {
"expected_count": expected_count,
"component": component,
"actual_count": count,
}
self.message = MoreThanCountException.MESSAGE % args
def check_health(self):
instances = self.cluster_context.get_instances(
node_process=self.process)
active_count = 0
for instance in instances:
status = self.process.status(instance)
if status == np.Status.RUNNING:
active_count += 1
if active_count == 0:
if self.process.ui_name in self.IMPORTANT_PROCESSES:
raise health_check_base.RedHealthError(
_("%s is not in running state") % self.process.ui_name)
else:
raise health_check_base.YellowHealthError(
_("%s is not in running state") % self.process.ui_name)
if active_count < len(instances):
if self.process.ui_name in self.IMPORTANT_PROCESSES:
raise health_check_base.YellowHealthError(
_("Some %s processes are not in running state") %
self.process.ui_name)
return _("%s is in running state") % self.process.ui_name
def post_install(self, cluster_context, instances):
hue_instance = cluster_context.get_instance(HUE)
@el.provision_event(name=_("Migrating Hue database"),
instance=hue_instance)
def migrate_database(remote, cluster_context):
hue_home = self.home_dir(cluster_context)
cmd = '%(activate)s && %(syncdb)s && %(migrate)s'
args = {
'activate': 'source %s/build/env/bin/activate' % hue_home,
'syncdb': '%s/build/env/bin/hue syncdb --noinput' % hue_home,
'migrate': '%s/build/env/bin/hue migrate' % hue_home,
}
remote.execute_command(cmd % args, run_as_root=True, timeout=600)
def hue_syncdb_workround(remote):
cmd = 'printf "/opt/mapr/lib\n$JAVA_HOME/jre/lib/amd64/server\n"' \
' | tee /etc/ld.so.conf.d/mapr-hue.conf && ldconfig'
remote.execute_command(cmd, run_as_root=True)
def centos7_workaround(remote):
cmd = 'ln -s /lib64/libsasl2.so.3.0.0 /lib64/libsasl2.so.2' \
' && rpm -ivh --nodeps http://yum.mariadb.org/5.5.49/' \
'rhel7-amd64/rpms/MariaDB-5.5.49-centos7-x86_64-compat.rpm'
remote.execute_command(cmd,
run_as_root=True,
raise_when_error=False)
with hue_instance.remote() as r:
LOG.debug("Executing Hue database migration")
# workaround for centos7
if cluster_context.distro_version.split('.')[0] == '7':
centos7_workaround(r)
# temporary workaround to prevent failure of db migrate on mapr 5.2
hue_syncdb_workround(r)
migrate_database(r, cluster_context)
self._copy_hive_configs(cluster_context, hue_instance)
self._install_jt_plugin(cluster_context, hue_instance)
self._set_service_dir_owner(cluster_context, instances)
class Sentry(s.Service):
SENTRY_STORAGE_MODE = p.Config(
name=SENTRY_MODE_CONFIG_NAME,
applicable_target='Sentry',
scope='cluster',
config_type="dropdown",
config_values=[
(v, v) for v in (FILE_STORAGE_SENTRY_MODE, DB_STORAGE_SENTRY_MODE)
],
priority=1,
description=_('Specifies Sentry storage mode.'))
GLOBAL_POLICY_FILE = '/user/mapr/sentry/global-policy.ini'
def __init__(self):
super(Sentry, self).__init__()
self._name = 'sentry'
self._ui_name = 'Sentry'
self._node_processes = [SENTRY]
self._priority = 2
def get_configs(self):
return [Sentry.SENTRY_STORAGE_MODE]
def get_config_files(self, cluster_context, configs, instance=None):
sentry_default = \
'plugins/mapr/services/sentry/resources/sentry-default.xml'
global_policy_template = \
'plugins/mapr/services/sentry/resources/global-policy.ini'
sentry_site = cf.HadoopXML('sentry-site.xml')
sentry_site.remote_path = self.conf_dir(cluster_context)
if instance:
sentry_site.fetch(instance)
sentry_site.load_properties(configs)
sentry_mode = configs[self.SENTRY_STORAGE_MODE.name]
sentry_site.parse(
utils.get_file_text(sentry_default, 'sahara_plugin_mapr'))
sentry_site.add_properties(
self._get_sentry_site_props(cluster_context, sentry_mode))
global_policy = cf.TemplateFile('global-policy.ini')
global_policy.remote_path = self.conf_dir(cluster_context)
global_policy.parse(
utils.get_file_text(global_policy_template, 'sahara_plugin_mapr'))
return [sentry_site, global_policy]
def _get_jdbc_uri(self, cluster_context):
jdbc_uri = ('jdbc:mysql://%(db_host)s:%(db_port)s/%(db_name)s?'
'createDatabaseIfNotExist=true')
jdbc_args = {
'db_host':
mysql.MySQL.get_db_instance(cluster_context).internal_ip,
'db_port': mysql.MySQL.MYSQL_SERVER_PORT,
'db_name': mysql.MySQL.SENTRY_SPECS.db_name,
}
return jdbc_uri % jdbc_args
def _get_sentry_site_props(self, cluster_context, setry_mode):
sentry_specs = mysql.MySQL.SENTRY_SPECS
if setry_mode == FILE_STORAGE_SENTRY_MODE:
return {
'sentry.hive.provider.backend':
'org.apache.sentry.provider'
'.file.SimpleFileProviderBackend',
'sentry.hive.provider.resource':
'maprfs:///' + self.GLOBAL_POLICY_FILE,
}
if setry_mode == DB_STORAGE_SENTRY_MODE:
return {
'sentry.hive.provider.backend':
'org.apache.sentry.provider.db.SimpleDBProviderBackend',
'sentry.store.jdbc.url': self._get_jdbc_uri(cluster_context),
'sentry.store.jdbc.driver': mysql.MySQL.DRIVER_CLASS,
'sentry.store.jdbc.user': sentry_specs.user,
'sentry.store.jdbc.password': sentry_specs.password,
}
def _init_db_schema(self, cluster_context):
instance = cluster_context.get_instance(SENTRY)
cmd = '%(home)s/bin/sentry --command schema-tool' \
' --conffile %(home)s/conf/sentry-site.xml' \
' --dbType mysql --initSchema' % {
'home': self.home_dir(cluster_context)}
@el.provision_event(name=_("Init Sentry DB schema"), instance=instance)
def decorated():
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
decorated()
def post_start(self, cluster_context, instances):
sentry_host = cluster_context.get_instance(SENTRY)
source = self.conf_dir(cluster_context) + '/global-policy.ini'
with sentry_host.remote() as r:
mfs.mkdir(r, '/user/mapr/sentry', run_as='mapr')
mfs.chmod(r, '/user/mapr/sentry', 777, run_as='mapr')
mfs.copy_from_local(r,
source,
self.GLOBAL_POLICY_FILE,
hdfs_user='******')
def _copy_warden_conf(self, cluster_context):
sentry_host = cluster_context.get_instance(SENTRY)
cmd = 'sudo -u mapr cp %s/conf.d/warden.sentry.conf' \
' /opt/mapr/conf/conf.d/' % self.home_dir(cluster_context)
with sentry_host.remote() as r:
r.execute_command(cmd)
def post_install(self, cluster_context, instances):
self._set_service_dir_owner(cluster_context, instances)
if cluster_context._get_cluster_config_value(
self.SENTRY_STORAGE_MODE) == DB_STORAGE_SENTRY_MODE:
self._init_db_schema(cluster_context)
self._copy_warden_conf(cluster_context)
def supports(self, service, mode):
"return True is Sentry supports integration"
service = service.name + '-' + service.version
return self.SENTRY_SUPPORT_MATRIX[mode][service]
class BaseConfigurer(ac.AbstractConfigurer, metaclass=abc.ABCMeta):
def configure(self, cluster_context, instances=None):
instances = instances or cluster_context.get_instances()
self._configure_ssh_connection(cluster_context, instances)
self._install_mapr_repo(cluster_context, instances)
if not cluster_context.is_prebuilt:
self._prepare_bare_image(cluster_context, instances)
self._install_services(cluster_context, instances)
if cluster_context.is_node_aware:
self._configure_topology(cluster_context, instances)
self._configure_database(cluster_context, instances)
self._configure_services(cluster_context, instances)
self._configure_sh_cluster(cluster_context, instances)
self._set_cluster_mode(cluster_context, instances)
self._post_configure_services(cluster_context, instances)
self._write_config_files(cluster_context, instances)
self._configure_environment(cluster_context, instances)
self._update_cluster_info(cluster_context)
def update(self, cluster_context, instances=None):
LOG.debug('Configuring existing instances')
instances = instances or cluster_context.get_instances()
existing = cluster_context.existing_instances()
if cluster_context.is_node_aware:
self._configure_topology(cluster_context, existing)
if cluster_context.has_control_nodes(instances):
self._configure_sh_cluster(cluster_context, existing)
self._post_configure_sh(cluster_context, existing)
self._write_config_files(cluster_context, existing)
self._update_services(cluster_context, existing)
self._restart_services(cluster_context)
self._update_cluster_info(cluster_context)
LOG.info('Existing instances successfully configured')
def _configure_services(self, cluster_context, instances):
for service in cluster_context.cluster_services:
service.configure(cluster_context, instances)
def _install_services(self, cluster_context, instances):
for service in self._service_install_sequence(cluster_context):
service.install(cluster_context, instances)
def _service_install_sequence(self, cluster_context):
def key(service):
if service in SERVICE_INSTALL_PRIORITY:
return SERVICE_INSTALL_PRIORITY.index(service)
return -service._priority
return sorted(cluster_context.cluster_services, key=key, reverse=True)
def _prepare_bare_image(self, cluster_context, instances):
LOG.debug('Preparing bare image')
if d.UBUNTU == cluster_context.distro:
self._install_security_repos(cluster_context, instances)
self._install_java(cluster_context, instances)
self._install_scala(cluster_context, instances)
self._install_mysql_client(cluster_context, instances)
LOG.debug('Bare images successfully prepared')
@el.provision_step(_("Install security repos"))
def _install_security_repos(self, cluster_context, instances):
LOG.debug("Installing security repos")
@el.provision_event()
def install_security_repos(instance):
return util.run_script(instance, ADD_SECURITY_REPO_SCRIPT, "root")
util.execute_on_instances(instances, install_security_repos)
@el.provision_step(_("Install MySQL client"))
def _install_mysql_client(self, cluster_context, instances):
LOG.debug("Installing MySQL client")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mysql_client(instance):
return util.run_script(instance, INSTALL_MYSQL_CLIENT,
"root", distro_name)
util.execute_on_instances(instances, install_mysql_client)
@el.provision_step(_("Install Scala"))
def _install_scala(self, cluster_context, instances):
LOG.debug("Installing Scala")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_scala(instance):
return util.run_script(instance, INSTALL_SCALA_SCRIPT,
"root", distro_name)
util.execute_on_instances(instances, install_scala)
@el.provision_step(_("Install Java"))
def _install_java(self, cluster_context, instances):
LOG.debug("Installing Java")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_java(instance):
return util.run_script(instance, INSTALL_JAVA_SCRIPT,
"root", distro_name)
util.execute_on_instances(instances, install_java)
@el.provision_step(_("Configure cluster topology"))
def _configure_topology(self, cluster_context, instances):
LOG.debug("Configuring cluster topology")
topology_map = cluster_context.topology_map
topology_map = ("%s %s" % item for item in topology_map.items())
topology_map = "\n".join(topology_map) + "\n"
data_path = "%s/topology.data" % cluster_context.mapr_home
script = utils.get_file_text(_TOPO_SCRIPT, 'sahara_plugin_mapr')
script_path = '%s/topology.sh' % cluster_context.mapr_home
@el.provision_event()
def write_topology_data(instance):
util.write_file(instance, data_path, topology_map, owner="root")
util.write_file(instance, script_path, script,
mode="+x", owner="root")
util.execute_on_instances(instances, write_topology_data)
LOG.info('Cluster topology successfully configured')
@el.provision_step(_("Write config files to instances"))
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for file in config_files:
util.write_file(instance, file.path, file.data, mode=file.mode,
owner="mapr")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances, write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _configure_environment(self, cluster_context, instances):
self.configure_general_environment(cluster_context, instances)
self._post_install_services(cluster_context, instances)
def _configure_database(self, cluster_context, instances):
mysql_instance = mysql.MySQL.get_db_instance(cluster_context)
@el.provision_event(instance=mysql_instance,
name=_("Configure database"))
def decorated():
distro_name = cluster_context.distro.name
distro_version = cluster_context.distro_version
mysql.MySQL.install_mysql(mysql_instance, distro_name,
distro_version)
mysql.MySQL.start_mysql_server(cluster_context)
mysql.MySQL.create_databases(cluster_context, instances)
decorated()
def _post_install_services(self, cluster_context, instances):
LOG.debug('Executing service post install hooks')
for s in cluster_context.cluster_services:
service_instances = cluster_context.filter_instances(instances,
service=s)
if service_instances:
s.post_install(cluster_context, instances)
LOG.info('Post install hooks execution successfully executed')
def _update_cluster_info(self, cluster_context):
LOG.debug('Updating UI information.')
info = {'Admin user credentials': {'Username': '******',
'Password': pu.get_mapr_password
(cluster_context.cluster)}}
for service in cluster_context.cluster_services:
for title, node_process, ui_info in (
service.get_ui_info(cluster_context)):
removed = cluster_context.removed_instances(node_process)
instances = cluster_context.get_instances(node_process)
instances = [i for i in instances if i not in removed]
if len(instances) == 1:
display_name_template = "%(title)s"
else:
display_name_template = "%(title)s %(index)s"
for index, instance in enumerate(instances, start=1):
args = {"title": title, "index": index}
display_name = display_name_template % args
data = ui_info.copy()
data[srvc.SERVICE_UI] = (data[srvc.SERVICE_UI] %
instance.get_ip_or_dns_name())
info.update({display_name: data})
ctx = context.ctx()
conductor.cluster_update(ctx, cluster_context.cluster, {'info': info})
def configure_general_environment(self, cluster_context, instances=None):
LOG.debug('Executing post configure hooks')
mapr_user_pass = pu.get_mapr_password(cluster_context.cluster)
if not instances:
instances = cluster_context.get_instances()
def set_user_password(instance):
LOG.debug('Setting password for user "mapr"')
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(
'echo "%s:%s"|chpasswd' %
('mapr', mapr_user_pass),
run_as_root=True)
else:
LOG.warning('User "mapr" does not exists')
def create_home_mapr(instance):
target_path = '/home/mapr'
LOG.debug("Creating home directory for user 'mapr'")
args = {'path': target_path,
'user': '******',
'group': 'mapr'}
cmd = ('mkdir -p %(path)s && chown %(user)s:%(group)s %(path)s'
% args)
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
else:
LOG.warning('User "mapr" does not exists')
util.execute_on_instances(instances, set_user_password)
util.execute_on_instances(instances, create_home_mapr)
@el.provision_step(_("Execute configure.sh"))
def _configure_sh_cluster(self, cluster_context, instances):
LOG.debug('Executing configure.sh')
if not instances:
instances = cluster_context.get_instances()
script = cluster_context.configure_sh
db_specs = dict(mysql.MySQL.METRICS_SPECS._asdict())
db_specs.update({
'host': mysql.MySQL.get_db_instance(cluster_context).internal_ip,
'port': mysql.MySQL.MYSQL_SERVER_PORT,
})
with context.PluginsThreadGroup() as tg:
for instance in instances:
tg.spawn('configure-sh-%s' % instance.id,
self._configure_sh_instance, cluster_context,
instance, script, db_specs)
LOG.debug('Executing configure.sh successfully completed')
@el.provision_event(instance_reference=2)
def _configure_sh_instance(self, cluster_context, instance, command,
specs):
if not self.mapr_user_exists(instance):
command += ' --create-user'
if cluster_context.check_for_process(instance, mng.METRICS):
command += (' -d %(host)s:%(port)s -du %(user)s -dp %(password)s '
'-ds %(db_name)s') % specs
with instance.remote() as r:
r.execute_command('sudo -i ' + command,
timeout=_CONFIGURE_SH_TIMEOUT)
@el.provision_step(_("Configure SSH connection"))
def _configure_ssh_connection(self, cluster_context, instances):
@el.provision_event()
def configure_ssh(instance):
echo_param = 'echo "KeepAlive yes" >> ~/.ssh/config'
echo_timeout = 'echo "ServerAliveInterval 60" >> ~/.ssh/config'
with instance.remote() as r:
r.execute_command(echo_param)
r.execute_command(echo_timeout)
util.execute_on_instances(instances, configure_ssh)
def mapr_user_exists(self, instance):
with instance.remote() as r:
ec, __ = r.execute_command(
"id -u %s" %
'mapr', run_as_root=True, raise_when_error=False)
return ec == 0
def post_start(self, cluster_context, instances=None):
instances = instances or cluster_context.get_instances()
LOG.debug('Executing service post start hooks')
for service in cluster_context.cluster_services:
updated = cluster_context.filter_instances(instances,
service=service)
service.post_start(cluster_context, updated)
LOG.info('Post start hooks successfully executed')
@el.provision_step(_("Set cluster mode"))
def _set_cluster_mode(self, cluster_context, instances):
cluster_mode = cluster_context.cluster_mode
if not cluster_mode:
return
command = "maprcli cluster mapreduce set -mode %s" % cluster_mode
@el.provision_event()
def set_cluster_mode(instance):
return util.execute_command([instance], command,
run_as='mapr')
util.execute_on_instances(instances, set_cluster_mode)
@el.provision_step(_("Install MapR repositories"))
def _install_mapr_repo(self, cluster_context, instances):
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mapr_repos(instance):
return util.run_script(instance, ADD_MAPR_REPO_SCRIPT, "root",
distro_name, **cluster_context.mapr_repos)
util.execute_on_instances(instances, install_mapr_repos)
def _update_services(self, cluster_context, instances):
for service in cluster_context.cluster_services:
updated = cluster_context.filter_instances(instances,
service=service)
service.update(cluster_context, updated)
def _restart_services(self, cluster_context):
restart = cluster_context.should_be_restarted
for service, instances in restart.items():
service.restart(util.unique_list(instances))
def _post_configure_sh(self, cluster_context, instances):
LOG.debug('Executing post configure.sh hooks')
for service in cluster_context.cluster_services:
service.post_configure_sh(cluster_context, instances)
LOG.info('Post configure.sh hooks successfully executed')
def _post_configure_services(self, cluster_context, instances):
for service in cluster_context.cluster_services:
service.post_configure(cluster_context, instances)
def _j2_render(template, arg_dict):
if template:
return template.render(arg_dict)
else:
raise e.PluginsInvalidDataException(
_('Template object must be defined'))
class MapRFS(s.Service):
_CREATE_DISK_LIST = 'plugins/mapr/resources/create_disk_list_file.sh'
_DISK_SETUP_CMD = '/opt/mapr/server/disksetup -F /tmp/disk.list'
_DISK_SETUP_TIMEOUT = 600
ENABLE_MAPR_DB_NAME = 'Enable MapR-DB'
HEAP_SIZE_PERCENT_NAME = 'MapR-FS heap size percent'
ENABLE_MAPR_DB_CONFIG = p.Config(
name=ENABLE_MAPR_DB_NAME,
applicable_target='general',
scope='cluster',
config_type="bool",
priority=1,
default_value=True,
description=_('Specifies that MapR-DB is in use.'))
HEAP_SIZE_PERCENT = p.Config(
name=HEAP_SIZE_PERCENT_NAME,
applicable_target='MapRFS',
scope='cluster',
config_type="int",
priority=1,
default_value=8,
description=_(
'Specifies heap size for MapR-FS in percents of maximum value.'))
def __init__(self):
super(MapRFS, self).__init__()
self._ui_name = 'MapRFS'
self._node_processes = [CLDB, FILE_SERVER, NFS]
self._ui_info = [
('Container Location Database (CLDB)', CLDB, {
s.SERVICE_UI: 'http://%s:7221'
}),
]
self._validation_rules = [
vu.at_least(1, CLDB),
vu.each_node_has(FILE_SERVER),
vu.on_same_node(CLDB, FILE_SERVER),
vu.has_volumes(),
]
def service_dir(self, cluster_context):
return
def home_dir(self, cluster_context):
return
def conf_dir(self, cluster_context):
return '%s/conf' % cluster_context.mapr_home
def post_install(self, cluster_context, instances):
LOG.debug('Initializing MapR FS')
instances = instances or cluster_context.get_instances()
file_servers = cluster_context.filter_instances(instances, FILE_SERVER)
utils.add_provisioning_step(cluster_context.cluster.id,
_("Initializing MapR-FS"),
len(file_servers))
with context.PluginsThreadGroup() as tg:
for instance in file_servers:
tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance,
instance)
LOG.info('MapR FS successfully initialized')
@el.provision_event(instance_reference=1)
def _init_mfs_instance(self, instance):
self._generate_disk_list_file(instance, self._CREATE_DISK_LIST)
self._execute_disksetup(instance)
def _generate_disk_list_file(self, instance, path_to_disk_setup_script):
LOG.debug('Creating disk list file')
g.run_script(instance, path_to_disk_setup_script, 'root',
*instance.storage_paths())
def _execute_disksetup(self, instance):
with instance.remote() as rmt:
rmt.execute_command(self._DISK_SETUP_CMD,
run_as_root=True,
timeout=self._DISK_SETUP_TIMEOUT)
def get_configs(self):
return [MapRFS.ENABLE_MAPR_DB_CONFIG, MapRFS.HEAP_SIZE_PERCENT]
def get_config_files(self, cluster_context, configs, instance=None):
default_path = 'plugins/mapr/services/maprfs/resources/cldb.conf'
cldb_conf = bcf.PropertiesFile("cldb.conf")
cldb_conf.remote_path = self.conf_dir(cluster_context)
if instance:
cldb_conf.fetch(instance)
cldb_conf.parse(utils.get_file_text(default_path,
'sahara_plugin_mapr'))
cldb_conf.add_properties(self._get_cldb_conf_props(cluster_context))
warden_conf = bcf.PropertiesFile("warden.conf")
warden_conf.remote_path = "/opt/mapr/conf/"
if instance:
warden_conf.fetch(instance)
warden_conf.add_properties({
'service.command.mfs.heapsize.percent':
configs[self.HEAP_SIZE_PERCENT_NAME]
})
return [cldb_conf, warden_conf]
def _get_cldb_conf_props(self, cluster_context):
zookeepers = cluster_context.get_zookeeper_nodes_ip_with_port()
return {'cldb.zookeeper.servers': zookeepers}
class Hue(s.Service):
THRIFT_VERSIONS = [5, 7]
THRIFT_VERSION = p.Config(name="Thrift version",
applicable_target="Hue",
scope='cluster',
config_type="dropdown",
config_values=[(v, v) for v in THRIFT_VERSIONS],
priority=1,
description=_('Specifies thrift version.'))
def __init__(self):
super(Hue, self).__init__()
self._name = 'hue'
self._ui_name = 'Hue'
self._node_processes = [HUE]
self._ui_info = None
self._validation_rules = [
vu.exactly(1, HUE),
vu.on_same_node(HUE, httpfs.HTTP_FS),
vu.on_same_node(HUE_LIVY, spark.SPARK_SLAVE),
]
self._priority = 2
def get_ui_info(self, cluster_context):
# Hue uses credentials of the administrative user (PAM auth)
return [('HUE', HUE, {
s.SERVICE_UI: 'http://%s:8888',
'Username': '******',
'Password': pu.get_mapr_password(cluster_context.cluster)
})]
def get_configs(self):
return [Hue.THRIFT_VERSION]
def conf_dir(self, cluster_context):
return '%s/desktop/conf' % self.home_dir(cluster_context)
def get_config_files(self, cluster_context, configs, instance=None):
template = 'plugins/mapr/services/hue/resources/hue_%s.template'
# hue.ini
hue_ini = bcf.TemplateFile("hue.ini")
hue_ini.remote_path = self.conf_dir(cluster_context)
hue_ini.parse(
utils.get_file_text(template % self.version, 'sahara_plugin_mapr'))
hue_ini.add_properties(self._get_hue_ini_props(cluster_context))
hue_ini.add_property("thrift_version",
configs[self.THRIFT_VERSION.name])
# # hue.sh
hue_sh_template = 'plugins/mapr/services/hue/' \
'resources/hue_sh_%s.template'
hue_sh = bcf.TemplateFile("hue.sh")
hue_sh.remote_path = self.home_dir(cluster_context) + '/bin'
hue_sh.parse(
utils.get_file_text(hue_sh_template % self.version,
'sahara_plugin_mapr'))
hue_sh.add_property('hadoop_version', cluster_context.hadoop_version)
hue_sh.mode = 777
hue_instances = cluster_context.get_instances(HUE)
for instance in hue_instances:
if instance not in cluster_context.changed_instances():
cluster_context.should_be_restarted[self] += [instance]
return [hue_ini, hue_sh]
def _get_hue_ini_props(self, cluster_context):
db_instance = mysql.MySQL.get_db_instance(cluster_context)
is_yarn = cluster_context.cluster_mode == 'yarn'
hue_specs = mysql.MySQL.HUE_SPECS
rdbms_specs = mysql.MySQL.RDBMS_SPECS
result = {
'db_host':
db_instance.internal_ip,
'hue_name':
hue_specs.db_name,
'hue_user':
hue_specs.user,
'hue_password':
hue_specs.password,
'rdbms_name':
rdbms_specs.db_name,
'rdbms_user':
rdbms_specs.user,
'rdbms_password':
rdbms_specs.password,
'resource_manager_uri':
cluster_context.resource_manager_uri,
'yarn_mode':
is_yarn,
'rm_host':
cluster_context.get_instance_ip(yarn.RESOURCE_MANAGER),
'webhdfs_url':
cluster_context.get_instance_ip(httpfs.HTTP_FS),
'jt_host':
cluster_context.get_instance_ip(mr.JOB_TRACKER),
'oozie_host':
cluster_context.get_instance_ip(oozie.OOZIE),
'sqoop_host':
cluster_context.get_instance_ip(sqoop.SQOOP_2_SERVER),
'impala_host':
cluster_context.get_instance_ip(impala.IMPALA_STATE_STORE),
'zk_hosts_with_port':
cluster_context.get_zookeeper_nodes_ip_with_port(),
'secret_key':
self._generate_secret_key()
}
hive_host = cluster_context.get_instance(hive.HIVE_SERVER_2)
if hive_host:
hive_service = cluster_context.get_service(hive.HIVE_SERVER_2)
result.update({
'hive_host':
hive_host.internal_ip,
'hive_version':
hive_service.version,
'hive_conf_dir':
hive_service.conf_dir(cluster_context),
})
hbase_host = cluster_context.get_instance(hbase.HBASE_THRIFT)
if hbase_host:
hbase_service = cluster_context.get_service(hbase.HBASE_THRIFT)
result.update({
'hbase_host':
hbase_host.internal_ip,
'hbase_conf_dir':
hbase_service.conf_dir(cluster_context),
})
livy_host = cluster_context.get_instance(HUE_LIVY)
if livy_host:
result.update({'livy_host': livy_host.internal_ip})
sentry_host = cluster_context.get_instance(sentry.SENTRY)
if sentry_host:
ui_name = sentry.Sentry().ui_name
sentry_version = cluster_context.get_chosen_service_version(
ui_name)
sentry_service = cluster_context. \
_find_service_instance(ui_name, sentry_version)
result.update({
'sentry_host':
sentry_host.internal_ip,
'sentry_conf':
sentry_service.conf_dir(cluster_context)
})
return result
def post_install(self, cluster_context, instances):
hue_instance = cluster_context.get_instance(HUE)
@el.provision_event(name=_("Migrating Hue database"),
instance=hue_instance)
def migrate_database(remote, cluster_context):
hue_home = self.home_dir(cluster_context)
cmd = '%(activate)s && %(syncdb)s && %(migrate)s'
args = {
'activate': 'source %s/build/env/bin/activate' % hue_home,
'syncdb': '%s/build/env/bin/hue syncdb --noinput' % hue_home,
'migrate': '%s/build/env/bin/hue migrate' % hue_home,
}
remote.execute_command(cmd % args, run_as_root=True, timeout=600)
def hue_syncdb_workround(remote):
cmd = 'printf "/opt/mapr/lib\n$JAVA_HOME/jre/lib/amd64/server\n"' \
' | tee /etc/ld.so.conf.d/mapr-hue.conf && ldconfig'
remote.execute_command(cmd, run_as_root=True)
def centos7_workaround(remote):
cmd = 'ln -s /lib64/libsasl2.so.3.0.0 /lib64/libsasl2.so.2' \
' && rpm -ivh --nodeps http://yum.mariadb.org/5.5.49/' \
'rhel7-amd64/rpms/MariaDB-5.5.49-centos7-x86_64-compat.rpm'
remote.execute_command(cmd,
run_as_root=True,
raise_when_error=False)
with hue_instance.remote() as r:
LOG.debug("Executing Hue database migration")
# workaround for centos7
if cluster_context.distro_version.split('.')[0] == '7':
centos7_workaround(r)
# temporary workaround to prevent failure of db migrate on mapr 5.2
hue_syncdb_workround(r)
migrate_database(r, cluster_context)
self._copy_hive_configs(cluster_context, hue_instance)
self._install_jt_plugin(cluster_context, hue_instance)
self._set_service_dir_owner(cluster_context, instances)
def _copy_hive_configs(self, cluster_context, hue_instance):
hive_server = cluster_context.get_instance(hive.HIVE_SERVER_2)
if not hive_server or hive_server == hue_instance:
LOG.debug('No Hive Servers found. Skip')
return
hive_service = cluster_context.get_service(hive.HIVE_SERVER_2)
hive_conf_dir = hive_service.conf_dir(cluster_context)
g.copy(hive_conf_dir, hive_server, hive_conf_dir, hue_instance, 'root')
def update(self, cluster_context, instances=None):
if self._should_restart(cluster_context, instances):
self.restart(instances)
def post_start(self, cluster_context, instances):
self.update(cluster_context, instances=instances)
def _should_restart(self, cluster_context, instances):
app_services = [
impala.Impala(),
hive.Hive(),
hbase.HBase(),
sqoop.Sqoop2(),
spark.SparkOnYarn(),
]
instances = [
cluster_context.filter_instances(instances, service=service)
for service in app_services
]
return bool(g.unique_list(itertools.chain(*instances)))
def jt_plugin_path(self, cluster_context):
path = ('%(home)s/desktop/libs/hadoop/java-lib'
'/hue-plugins-%(version)s-mapr.jar')
args = {
'home': self.home_dir(cluster_context),
'version': self.version,
}
return path % args
@el.provision_event(name="Install Hue Job Tracker plugin",
instance_reference=2)
def _install_jt_plugin(self, cluster_context, hue_instance):
LOG.debug("Copying Hue JobTracker plugin")
job_trackers = cluster_context.get_instances(mr.JOB_TRACKER)
if not job_trackers:
LOG.debug('No JobTrackers found. Skip')
return
jt_plugin_src = self.jt_plugin_path(cluster_context)
jt_plugin_dest = cluster_context.hadoop_lib + '/jt_plugin.jar'
for jt in job_trackers:
g.copy(jt_plugin_src, hue_instance, jt_plugin_dest, jt, 'root')
def _generate_secret_key(self, length=80):
ascii_alphanum = string.ascii_letters + string.digits
generator = random.SystemRandom()
return ''.join(generator.choice(ascii_alphanum) for _ in range(length))
class MapRPlugin(p.ProvisioningPluginBase):
title = 'MapR Hadoop Distribution'
description = _('The MapR Distribution provides a full Hadoop stack that'
' includes the MapR File System (MapR-FS), MapReduce,'
' a complete Hadoop ecosystem, and the MapR Control System'
' user interface')
def _get_handler(self, hadoop_version):
return vhf.VersionHandlerFactory.get().get_handler(hadoop_version)
def get_title(self):
return MapRPlugin.title
def get_description(self):
return MapRPlugin.description
def get_labels(self):
return {
'plugin_labels': {'enabled': {'status': True}},
'version_labels': {
'5.2.0.mrv2': {'enabled': {'status': True}},
}
}
def get_versions(self):
return vhf.VersionHandlerFactory.get().get_versions()
def get_node_processes(self, hadoop_version):
return self._get_handler(hadoop_version).get_node_processes()
def get_configs(self, hadoop_version):
return self._get_handler(hadoop_version).get_configs()
def configure_cluster(self, cluster):
self._get_handler(cluster.hadoop_version).configure_cluster(cluster)
def start_cluster(self, cluster):
self._get_handler(cluster.hadoop_version).start_cluster(cluster)
def validate(self, cluster):
self._get_handler(cluster.hadoop_version).validate(cluster)
def validate_scaling(self, cluster, existing, additional):
v_handler = self._get_handler(cluster.hadoop_version)
v_handler.validate_scaling(cluster, existing, additional)
def scale_cluster(self, cluster, instances):
v_handler = self._get_handler(cluster.hadoop_version)
v_handler.scale_cluster(cluster, instances)
def decommission_nodes(self, cluster, instances):
v_handler = self._get_handler(cluster.hadoop_version)
v_handler.decommission_nodes(cluster, instances)
def get_edp_engine(self, cluster, job_type):
v_handler = self._get_handler(cluster.hadoop_version)
return v_handler.get_edp_engine(cluster, job_type)
def get_edp_job_types(self, versions=None):
res = {}
for vers in self.get_versions():
if not versions or vers in versions:
vh = self._get_handler(vers)
res[vers] = vh.get_edp_job_types()
return res
def get_edp_config_hints(self, job_type, version):
v_handler = self._get_handler(version)
return v_handler.get_edp_config_hints(job_type)
def get_open_ports(self, node_group):
v_handler = self._get_handler(node_group.cluster.hadoop_version)
return v_handler.get_open_ports(node_group)
def get_health_checks(self, cluster):
v_handler = self._get_handler(cluster.hadoop_version)
return v_handler.get_cluster_checks(cluster)
def get_image_arguments(self, hadoop_version):
return self._get_handler(hadoop_version).get_image_arguments()
def pack_image(self, hadoop_version, remote,
test_only=False, image_arguments=None):
version = self._get_handler(hadoop_version)
version.pack_image(hadoop_version, remote, test_only=test_only,
image_arguments=image_arguments)
def validate_images(self, cluster, test_only=False, image_arguments=None):
self._get_handler(cluster.hadoop_version).validate_images(
cluster, test_only=test_only, image_arguments=image_arguments)
class Oozie(s.Service):
def __init__(self):
super(Oozie, self).__init__()
self._name = 'oozie'
self._ui_name = 'Oozie'
self._node_processes = [OOZIE]
self._cluster_defaults = ['oozie-default.json']
self._validation_rules = [vu.exactly(1, OOZIE)]
self._ui_info = [('Oozie', OOZIE, {
s.SERVICE_UI: 'http://%s:11000/oozie'
})]
def libext_path(self):
return '/opt/mapr/oozie/oozie-%s/oozie-server/lib/' % self.version
def get_config_files(self, cluster_context, configs, instance=None):
oozie_site = bcf.HadoopXML("oozie-site.xml")
oozie_site.remote_path = self.conf_dir(cluster_context)
if instance:
oozie_site.fetch(instance)
oozie_site.load_properties(configs)
oozie_site.add_properties(self._get_oozie_site_props(cluster_context))
return [oozie_site]
def _get_oozie_site_props(self, cluster_context):
oozie_specs = mysql.MySQL.OOZIE_SPECS
return {
'oozie.db.schema.name':
oozie_specs.db_name,
'oozie.service.JPAService.create.db.schema':
True,
'oozie.service.JPAService.jdbc.driver':
mysql.MySQL.DRIVER_CLASS,
'oozie.service.JPAService.jdbc.url':
self._get_jdbc_uri(cluster_context),
'oozie.service.JPAService.jdbc.username':
oozie_specs.user,
'oozie.service.JPAService.jdbc.password':
oozie_specs.password,
'oozie.service.HadoopAccessorService.hadoop.configurations':
'*=%s' % cluster_context.hadoop_conf
}
def _get_jdbc_uri(self, cluster_context):
jdbc_uri = ('jdbc:mysql://%(db_host)s:%(db_port)s/%(db_name)s?'
'createDatabaseIfNotExist=true')
jdbc_args = {
'db_host':
mysql.MySQL.get_db_instance(cluster_context).internal_ip,
'db_port': mysql.MySQL.MYSQL_SERVER_PORT,
'db_name': mysql.MySQL.OOZIE_SPECS.db_name,
}
return jdbc_uri % jdbc_args
def install(self, cluster_context, instances):
# oozie requires executed configure.sh
pass
def post_configure(self, cluster_context, instances):
super(Oozie, self).install(cluster_context, instances)
oozie_instances = cluster_context.filter_instances(instances,
service=self)
for instance in oozie_instances:
with instance.remote() as r:
r.execute_command(
'sudo cp '
'/opt/mapr/oozie/oozie-%s/conf/warden.oozie.conf '
'/opt/mapr/conf/conf.d' % self.version)
def post_install(self, cluster_context, instances):
oozie_inst = cluster_context.get_instance(OOZIE)
oozie_service = cluster_context.get_service(OOZIE)
if oozie_service:
symlink_cmd = ('cp /opt/mapr/lib/mysql-connector-*.jar %s' %
self.libext_path())
with oozie_inst.remote() as r:
LOG.debug('Installing MySQL connector for Oozie')
r.execute_command(symlink_cmd,
run_as_root=True,
raise_when_error=False)
self._set_service_dir_owner(cluster_context, instances)
def post_start(self, cluster_context, instances):
instances = cluster_context.filter_instances(instances, OOZIE)
self._rebuild(cluster_context, instances)
@el.provision_event(instance_reference=1)
@g.remote_command(1)
def _rebuild_oozie_war(self, remote, cluster_context):
cmd = 'cp -f /opt/mapr-repository/ext-2.2.zip ' \
'%(home)s/libext/ext-2.2.zip &&' \
' %(home)s/bin/oozie-setup.sh -hadoop %(version)s' \
' /opt/mapr/hadoop/hadoop-%(version)s'
args = {
'home': self.home_dir(cluster_context),
'version': cluster_context.hadoop_version
}
remote.execute_command(cmd % args, run_as_root=True)
def update(self, cluster_context, instances=None):
instances = instances or cluster_context.get_instances()
instances = cluster_context.filter_instances(instances, OOZIE)
self._rebuild(cluster_context, instances)
@el.provision_step(_("Rebuilt Oozie war"))
def _rebuild(self, cluster_context, instances):
OOZIE.stop(list(filter(OOZIE.is_started, instances)))
g.execute_on_instances(instances, self._rebuild_oozie_war,
cluster_context)
OOZIE.start(instances)
context.sleep(OOZIE_START_DELAY)
