def _create_config_obj(self, item, target='general', scope='cluster', high_priority=False): def _prepare_value(value): if isinstance(value, str): return value.strip().lower() return value conf_name = _prepare_value(item.get('name', None)) conf_value = _prepare_value(item.get('value', None)) if not conf_name: raise ex.HadoopProvisionError(_("Config missing 'name'")) if conf_value is None: raise ex.PluginInvalidDataException( _("Config '%s' missing 'value'") % conf_name) if high_priority or item.get('priority', 2) == 1: priority = 1 else: priority = 2 return p.Config(name=conf_name, applicable_target=target, scope=scope, config_type=item.get('config_type', "string"), config_values=item.get('config_values', None), default_value=conf_value, is_optional=item.get('is_optional', True), description=item.get('description', None), priority=priority)
def _await_cldb(self, cluster_context, instances=None, timeout=600): instances = instances or cluster_context.get_instances() cldb_node = cluster_context.get_instance(mfs.CLDB) start_time = timeutils.utcnow() retry_count = 0 with cldb_node.remote() as r: LOG.debug("Waiting {count} seconds for CLDB initialization".format( count=timeout)) while timeutils.delta_seconds(start_time, timeutils.utcnow()) < timeout: ec, out = r.execute_command(NODE_LIST_CMD, raise_when_error=False) resp = json.loads(out) status = resp['status'] if str(status).lower() == 'ok': ips = [n['ip'] for n in resp['data']] retry_count += 1 for i in instances: if (i.internal_ip not in ips and (retry_count > DEFAULT_RETRY_COUNT)): msg = _("Node failed to connect to CLDB: %s" ) % i.internal_ip raise ex.HadoopProvisionError(msg) break else: context.sleep(DELAY) else: raise ex.HadoopProvisionError(_("CLDB failed to start"))
def stop(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() zookeepers = cluster_context.filter_instances(instances, mng.ZOOKEEPER) utils.add_provisioning_step(cluster_context.cluster.id, _("Stop ZooKeepers nodes"), len(zookeepers)) self._stop_zk_nodes(zookeepers) utils.add_provisioning_step(cluster_context.cluster.id, _("Stop Warden nodes"), len(instances)) self._stop_warden_on_nodes(instances)
def get_service(self, node_process): ui_name = self.get_service_name_by_node_process(node_process) if ui_name is None: raise e.PluginInvalidDataException( _('Service not found in services list')) version = self.get_chosen_service_version(ui_name) service = self._find_service_instance(ui_name, version) if service is None: raise e.PluginInvalidDataException(_('Can not map service')) return service
class NodeRequiredServiceMissingException(e.RequiredServiceMissingException): MISSING_MSG = _('Node "%(ng_name)s" is missing component %(component)s') REQUIRED_MSG = _('%(message)s, required by %(required_by)s') def __init__(self, service_name, ng_name, required_by=None): super(NodeRequiredServiceMissingException, self).__init__(service_name, required_by) args = {'ng_name': ng_name, 'component': service_name} self.message = (NodeRequiredServiceMissingException.MISSING_MSG % args) if required_by: args = {'message': self.message, 'required_by': required_by} self.message = (NodeRequiredServiceMissingException.REQUIRED_MSG % args)
def check_health(self): instances = self.cluster_context.get_instances( node_process=management.ZOOKEEPER) active_count = 0 for instance in instances: if self._is_zookeeper_running(instance): active_count += 1 if active_count == 0: raise health_check_base.RedHealthError( _("ZooKeeper is not in running state")) if active_count < len(instances): raise health_check_base.YellowHealthError( _("Some ZooKeeper processes are not in running state")) return _("ZooKeeper is in running state")
class NoVolumesException(e.SaharaPluginException): MESSAGE = _('%s must have at least 1 volume or ephemeral drive') ERROR_CODE = "NO_VOLUMES" def __init__(self, ng_name): super(NoVolumesException, self).__init__() self.message = NoVolumesException.MESSAGE % ng_name self.code = NoVolumesException.ERROR_CODE
class EvenCountException(e.SaharaPluginException): MESSAGE = _("Hadoop cluster should contain odd number of %(component)s" " but %(actual_count)s found.") def __init__(self, component, count): super(EvenCountException, self).__init__() args = {'component': component, 'actual_count': count} self.message = EvenCountException.MESSAGE % args
def get_node_process_name(node_process): # This import is placed here to avoid circular imports from sahara_plugin_mapr.plugins.mapr.domain import node_process as np # noqa if isinstance(node_process, np.NodeProcess): return node_process.ui_name if isinstance(node_process, six.string_types): return node_process raise TypeError(_("Invalid argument type %s") % type(node_process))
class NotRequiredImageException(e.SaharaPluginException): MESSAGE = _('Service %(service)s requires %(os)s OS.' ' Use %(os)s image and add "%(os)s" tag to it.') ERROR_CODE = "INVALID_IMAGE" def __init__(self, service, os): super(NotRequiredImageException, self).__init__() self.message = NotRequiredImageException.MESSAGE % {'service': service, 'os': os} self.code = NotRequiredImageException.ERROR_CODE
def get_version_config(self, versions): return p.Config(name='%s Version' % self._ui_name, applicable_target=self.ui_name, scope='cluster', config_type='dropdown', config_values=[ (v, v) for v in sorted(versions, reverse=True) ], is_optional=False, description=_('Specify the version of the service'), priority=1)
def start(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() zookeepers = cluster_context.filter_instances(instances, mng.ZOOKEEPER) cldbs = cluster_context.filter_instances(instances, mfs.CLDB) others = filter( lambda i: not cluster_context.check_for_process(i, mfs.CLDB), instances) utils.add_provisioning_step(cluster_context.cluster.id, _("Start ZooKeepers nodes"), len(zookeepers)) self._start_zk_nodes(zookeepers) utils.add_provisioning_step(cluster_context.cluster.id, _("Start CLDB nodes"), len(cldbs)) self._start_cldb_nodes(cldbs) if others: utils.add_provisioning_step(cluster_context.cluster.id, _("Start non-CLDB nodes"), len(list(others))) self._start_non_cldb_nodes(others) self._await_cldb(cluster_context, instances)
def post_install(self, cluster_context, instances): LOG.debug('Initializing MapR FS') instances = instances or cluster_context.get_instances() file_servers = cluster_context.filter_instances(instances, FILE_SERVER) utils.add_provisioning_step(cluster_context.cluster.id, _("Initializing MapR-FS"), len(file_servers)) with context.PluginsThreadGroup() as tg: for instance in file_servers: tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance, instance) LOG.info('MapR FS successfully initialized')
class NodeServiceConflictException(e.SaharaPluginException): MESSAGE = _('%(service)s service cannot be installed alongside' ' %(package)s package') ERROR_CODE = "NODE_PROCESS_CONFLICT" def __init__(self, service_name, conflicted_package): super(NodeServiceConflictException, self).__init__() args = { 'service': service_name, 'package': conflicted_package, } self.message = NodeServiceConflictException.MESSAGE % args self.code = NodeServiceConflictException.ERROR_CODE
def _init_db_schema(self, cluster_context): instance = cluster_context.get_instance(SENTRY) cmd = '%(home)s/bin/sentry --command schema-tool' \ ' --conffile %(home)s/conf/sentry-site.xml' \ ' --dbType mysql --initSchema' % { 'home': self.home_dir(cluster_context)} @el.provision_event(name=_("Init Sentry DB schema"), instance=instance) def decorated(): with instance.remote() as r: r.execute_command(cmd, run_as_root=True) decorated()
def _get_repo_configs(self): ubuntu_base = p.Config( name="Ubuntu base repo", applicable_target="general", scope='cluster', priority=1, default_value="", description=_( 'Specifies Ubuntu MapR core repository.') ) centos_base = p.Config( name="CentOS base repo", applicable_target="general", scope='cluster', priority=1, default_value="", description=_( 'Specifies CentOS MapR core repository.') ) ubuntu_eco = p.Config( name="Ubuntu ecosystem repo", applicable_target="general", scope='cluster', priority=1, default_value="", description=_( 'Specifies Ubuntu MapR ecosystem repository.') ) centos_eco = p.Config( name="CentOS ecosystem repo", applicable_target="general", scope='cluster', priority=1, default_value="", description=_( 'Specifies CentOS MapR ecosystem repository.') ) return [ubuntu_base, centos_base, ubuntu_eco, centos_eco]
class LessThanCountException(e.InvalidComponentCountException): MESSAGE = _("Hadoop cluster should contain at least" " %(expected_count)d %(component)s component(s)." " Actual %(component)s count is %(actual_count)d") def __init__(self, component, expected_count, count): super(LessThanCountException, self).__init__( component, expected_count, count) args = { 'expected_count': expected_count, 'component': component, 'actual_count': count, } self.message = LessThanCountException.MESSAGE % args
def install(self, cluster_context, instances): service_instances = cluster_context.filter_instances(instances, service=self) @el.provision_step(_("Install %s service") % self.ui_name, cluster_context_reference=0, instances_reference=1) def _install(_context, _instances): g.execute_on_instances(_instances, self._install_packages_on_instance, _context) if service_instances: _install(cluster_context, service_instances)
def poll_status(instance): operation_name = _('Wait for {node_process} on {instance}' ' to change status to "{status}"') args = { 'node_process': self.ui_name, 'instance': instance.instance_name, 'status': status.name, } return plugin_utils.poll( get_status=lambda: self.status(instance) == status, operation_name=operation_name.format(**args), timeout=timeout, sleep=sleep, )
def _configure_database(self, cluster_context, instances): mysql_instance = mysql.MySQL.get_db_instance(cluster_context) @el.provision_event(instance=mysql_instance, name=_("Configure database")) def decorated(): distro_name = cluster_context.distro.name distro_version = cluster_context.distro_version mysql.MySQL.install_mysql(mysql_instance, distro_name, distro_version) mysql.MySQL.start_mysql_server(cluster_context) mysql.MySQL.create_databases(cluster_context, instances) decorated()
class MoreThanCountException(e.InvalidComponentCountException): MESSAGE = _("Hadoop cluster should contain at most" " %(expected_count)d %(component)s component(s)." " Actual %(component)s count is %(actual_count)d") def __init__(self, component, expected_count, count): super(MoreThanCountException, self).__init__( component, expected_count, count) args = { "expected_count": expected_count, "component": component, "actual_count": count, } self.message = MoreThanCountException.MESSAGE % args
def check_health(self): instances = self.cluster_context.get_instances( node_process=self.process) active_count = 0 for instance in instances: status = self.process.status(instance) if status == np.Status.RUNNING: active_count += 1 if active_count == 0: if self.process.ui_name in self.IMPORTANT_PROCESSES: raise health_check_base.RedHealthError( _("%s is not in running state") % self.process.ui_name) else: raise health_check_base.YellowHealthError( _("%s is not in running state") % self.process.ui_name) if active_count < len(instances): if self.process.ui_name in self.IMPORTANT_PROCESSES: raise health_check_base.YellowHealthError( _("Some %s processes are not in running state") % self.process.ui_name) return _("%s is in running state") % self.process.ui_name
def post_install(self, cluster_context, instances): hue_instance = cluster_context.get_instance(HUE) @el.provision_event(name=_("Migrating Hue database"), instance=hue_instance) def migrate_database(remote, cluster_context): hue_home = self.home_dir(cluster_context) cmd = '%(activate)s && %(syncdb)s && %(migrate)s' args = { 'activate': 'source %s/build/env/bin/activate' % hue_home, 'syncdb': '%s/build/env/bin/hue syncdb --noinput' % hue_home, 'migrate': '%s/build/env/bin/hue migrate' % hue_home, } remote.execute_command(cmd % args, run_as_root=True, timeout=600) def hue_syncdb_workround(remote): cmd = 'printf "/opt/mapr/lib\n$JAVA_HOME/jre/lib/amd64/server\n"' \ ' | tee /etc/ld.so.conf.d/mapr-hue.conf && ldconfig' remote.execute_command(cmd, run_as_root=True) def centos7_workaround(remote): cmd = 'ln -s /lib64/libsasl2.so.3.0.0 /lib64/libsasl2.so.2' \ ' && rpm -ivh --nodeps http://yum.mariadb.org/5.5.49/' \ 'rhel7-amd64/rpms/MariaDB-5.5.49-centos7-x86_64-compat.rpm' remote.execute_command(cmd, run_as_root=True, raise_when_error=False) with hue_instance.remote() as r: LOG.debug("Executing Hue database migration") # workaround for centos7 if cluster_context.distro_version.split('.')[0] == '7': centos7_workaround(r) # temporary workaround to prevent failure of db migrate on mapr 5.2 hue_syncdb_workround(r) migrate_database(r, cluster_context) self._copy_hive_configs(cluster_context, hue_instance) self._install_jt_plugin(cluster_context, hue_instance) self._set_service_dir_owner(cluster_context, instances)
class Sentry(s.Service): SENTRY_STORAGE_MODE = p.Config( name=SENTRY_MODE_CONFIG_NAME, applicable_target='Sentry', scope='cluster', config_type="dropdown", config_values=[ (v, v) for v in (FILE_STORAGE_SENTRY_MODE, DB_STORAGE_SENTRY_MODE) ], priority=1, description=_('Specifies Sentry storage mode.')) GLOBAL_POLICY_FILE = '/user/mapr/sentry/global-policy.ini' def __init__(self): super(Sentry, self).__init__() self._name = 'sentry' self._ui_name = 'Sentry' self._node_processes = [SENTRY] self._priority = 2 def get_configs(self): return [Sentry.SENTRY_STORAGE_MODE] def get_config_files(self, cluster_context, configs, instance=None): sentry_default = \ 'plugins/mapr/services/sentry/resources/sentry-default.xml' global_policy_template = \ 'plugins/mapr/services/sentry/resources/global-policy.ini' sentry_site = cf.HadoopXML('sentry-site.xml') sentry_site.remote_path = self.conf_dir(cluster_context) if instance: sentry_site.fetch(instance) sentry_site.load_properties(configs) sentry_mode = configs[self.SENTRY_STORAGE_MODE.name] sentry_site.parse( utils.get_file_text(sentry_default, 'sahara_plugin_mapr')) sentry_site.add_properties( self._get_sentry_site_props(cluster_context, sentry_mode)) global_policy = cf.TemplateFile('global-policy.ini') global_policy.remote_path = self.conf_dir(cluster_context) global_policy.parse( utils.get_file_text(global_policy_template, 'sahara_plugin_mapr')) return [sentry_site, global_policy] def _get_jdbc_uri(self, cluster_context): jdbc_uri = ('jdbc:mysql://%(db_host)s:%(db_port)s/%(db_name)s?' 'createDatabaseIfNotExist=true') jdbc_args = { 'db_host': mysql.MySQL.get_db_instance(cluster_context).internal_ip, 'db_port': mysql.MySQL.MYSQL_SERVER_PORT, 'db_name': mysql.MySQL.SENTRY_SPECS.db_name, } return jdbc_uri % jdbc_args def _get_sentry_site_props(self, cluster_context, setry_mode): sentry_specs = mysql.MySQL.SENTRY_SPECS if setry_mode == FILE_STORAGE_SENTRY_MODE: return { 'sentry.hive.provider.backend': 'org.apache.sentry.provider' '.file.SimpleFileProviderBackend', 'sentry.hive.provider.resource': 'maprfs:///' + self.GLOBAL_POLICY_FILE, } if setry_mode == DB_STORAGE_SENTRY_MODE: return { 'sentry.hive.provider.backend': 'org.apache.sentry.provider.db.SimpleDBProviderBackend', 'sentry.store.jdbc.url': self._get_jdbc_uri(cluster_context), 'sentry.store.jdbc.driver': mysql.MySQL.DRIVER_CLASS, 'sentry.store.jdbc.user': sentry_specs.user, 'sentry.store.jdbc.password': sentry_specs.password, } def _init_db_schema(self, cluster_context): instance = cluster_context.get_instance(SENTRY) cmd = '%(home)s/bin/sentry --command schema-tool' \ ' --conffile %(home)s/conf/sentry-site.xml' \ ' --dbType mysql --initSchema' % { 'home': self.home_dir(cluster_context)} @el.provision_event(name=_("Init Sentry DB schema"), instance=instance) def decorated(): with instance.remote() as r: r.execute_command(cmd, run_as_root=True) decorated() def post_start(self, cluster_context, instances): sentry_host = cluster_context.get_instance(SENTRY) source = self.conf_dir(cluster_context) + '/global-policy.ini' with sentry_host.remote() as r: mfs.mkdir(r, '/user/mapr/sentry', run_as='mapr') mfs.chmod(r, '/user/mapr/sentry', 777, run_as='mapr') mfs.copy_from_local(r, source, self.GLOBAL_POLICY_FILE, hdfs_user='******') def _copy_warden_conf(self, cluster_context): sentry_host = cluster_context.get_instance(SENTRY) cmd = 'sudo -u mapr cp %s/conf.d/warden.sentry.conf' \ ' /opt/mapr/conf/conf.d/' % self.home_dir(cluster_context) with sentry_host.remote() as r: r.execute_command(cmd) def post_install(self, cluster_context, instances): self._set_service_dir_owner(cluster_context, instances) if cluster_context._get_cluster_config_value( self.SENTRY_STORAGE_MODE) == DB_STORAGE_SENTRY_MODE: self._init_db_schema(cluster_context) self._copy_warden_conf(cluster_context) def supports(self, service, mode): "return True is Sentry supports integration" service = service.name + '-' + service.version return self.SENTRY_SUPPORT_MATRIX[mode][service]
class BaseConfigurer(ac.AbstractConfigurer, metaclass=abc.ABCMeta): def configure(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() self._configure_ssh_connection(cluster_context, instances) self._install_mapr_repo(cluster_context, instances) if not cluster_context.is_prebuilt: self._prepare_bare_image(cluster_context, instances) self._install_services(cluster_context, instances) if cluster_context.is_node_aware: self._configure_topology(cluster_context, instances) self._configure_database(cluster_context, instances) self._configure_services(cluster_context, instances) self._configure_sh_cluster(cluster_context, instances) self._set_cluster_mode(cluster_context, instances) self._post_configure_services(cluster_context, instances) self._write_config_files(cluster_context, instances) self._configure_environment(cluster_context, instances) self._update_cluster_info(cluster_context) def update(self, cluster_context, instances=None): LOG.debug('Configuring existing instances') instances = instances or cluster_context.get_instances() existing = cluster_context.existing_instances() if cluster_context.is_node_aware: self._configure_topology(cluster_context, existing) if cluster_context.has_control_nodes(instances): self._configure_sh_cluster(cluster_context, existing) self._post_configure_sh(cluster_context, existing) self._write_config_files(cluster_context, existing) self._update_services(cluster_context, existing) self._restart_services(cluster_context) self._update_cluster_info(cluster_context) LOG.info('Existing instances successfully configured') def _configure_services(self, cluster_context, instances): for service in cluster_context.cluster_services: service.configure(cluster_context, instances) def _install_services(self, cluster_context, instances): for service in self._service_install_sequence(cluster_context): service.install(cluster_context, instances) def _service_install_sequence(self, cluster_context): def key(service): if service in SERVICE_INSTALL_PRIORITY: return SERVICE_INSTALL_PRIORITY.index(service) return -service._priority return sorted(cluster_context.cluster_services, key=key, reverse=True) def _prepare_bare_image(self, cluster_context, instances): LOG.debug('Preparing bare image') if d.UBUNTU == cluster_context.distro: self._install_security_repos(cluster_context, instances) self._install_java(cluster_context, instances) self._install_scala(cluster_context, instances) self._install_mysql_client(cluster_context, instances) LOG.debug('Bare images successfully prepared') @el.provision_step(_("Install security repos")) def _install_security_repos(self, cluster_context, instances): LOG.debug("Installing security repos") @el.provision_event() def install_security_repos(instance): return util.run_script(instance, ADD_SECURITY_REPO_SCRIPT, "root") util.execute_on_instances(instances, install_security_repos) @el.provision_step(_("Install MySQL client")) def _install_mysql_client(self, cluster_context, instances): LOG.debug("Installing MySQL client") distro_name = cluster_context.distro.name @el.provision_event() def install_mysql_client(instance): return util.run_script(instance, INSTALL_MYSQL_CLIENT, "root", distro_name) util.execute_on_instances(instances, install_mysql_client) @el.provision_step(_("Install Scala")) def _install_scala(self, cluster_context, instances): LOG.debug("Installing Scala") distro_name = cluster_context.distro.name @el.provision_event() def install_scala(instance): return util.run_script(instance, INSTALL_SCALA_SCRIPT, "root", distro_name) util.execute_on_instances(instances, install_scala) @el.provision_step(_("Install Java")) def _install_java(self, cluster_context, instances): LOG.debug("Installing Java") distro_name = cluster_context.distro.name @el.provision_event() def install_java(instance): return util.run_script(instance, INSTALL_JAVA_SCRIPT, "root", distro_name) util.execute_on_instances(instances, install_java) @el.provision_step(_("Configure cluster topology")) def _configure_topology(self, cluster_context, instances): LOG.debug("Configuring cluster topology") topology_map = cluster_context.topology_map topology_map = ("%s %s" % item for item in topology_map.items()) topology_map = "\n".join(topology_map) + "\n" data_path = "%s/topology.data" % cluster_context.mapr_home script = utils.get_file_text(_TOPO_SCRIPT, 'sahara_plugin_mapr') script_path = '%s/topology.sh' % cluster_context.mapr_home @el.provision_event() def write_topology_data(instance): util.write_file(instance, data_path, topology_map, owner="root") util.write_file(instance, script_path, script, mode="+x", owner="root") util.execute_on_instances(instances, write_topology_data) LOG.info('Cluster topology successfully configured') @el.provision_step(_("Write config files to instances")) def _write_config_files(self, cluster_context, instances): LOG.debug('Writing config files') @el.provision_event() def write_config_files(instance, config_files): for file in config_files: util.write_file(instance, file.path, file.data, mode=file.mode, owner="mapr") node_groups = util.unique_list(instances, lambda i: i.node_group) for node_group in node_groups: config_files = cluster_context.get_config_files(node_group) ng_instances = [i for i in node_group.instances if i in instances] util.execute_on_instances(ng_instances, write_config_files, config_files=config_files) LOG.debug("Config files are successfully written") def _configure_environment(self, cluster_context, instances): self.configure_general_environment(cluster_context, instances) self._post_install_services(cluster_context, instances) def _configure_database(self, cluster_context, instances): mysql_instance = mysql.MySQL.get_db_instance(cluster_context) @el.provision_event(instance=mysql_instance, name=_("Configure database")) def decorated(): distro_name = cluster_context.distro.name distro_version = cluster_context.distro_version mysql.MySQL.install_mysql(mysql_instance, distro_name, distro_version) mysql.MySQL.start_mysql_server(cluster_context) mysql.MySQL.create_databases(cluster_context, instances) decorated() def _post_install_services(self, cluster_context, instances): LOG.debug('Executing service post install hooks') for s in cluster_context.cluster_services: service_instances = cluster_context.filter_instances(instances, service=s) if service_instances: s.post_install(cluster_context, instances) LOG.info('Post install hooks execution successfully executed') def _update_cluster_info(self, cluster_context): LOG.debug('Updating UI information.') info = {'Admin user credentials': {'Username': '******', 'Password': pu.get_mapr_password (cluster_context.cluster)}} for service in cluster_context.cluster_services: for title, node_process, ui_info in ( service.get_ui_info(cluster_context)): removed = cluster_context.removed_instances(node_process) instances = cluster_context.get_instances(node_process) instances = [i for i in instances if i not in removed] if len(instances) == 1: display_name_template = "%(title)s" else: display_name_template = "%(title)s %(index)s" for index, instance in enumerate(instances, start=1): args = {"title": title, "index": index} display_name = display_name_template % args data = ui_info.copy() data[srvc.SERVICE_UI] = (data[srvc.SERVICE_UI] % instance.get_ip_or_dns_name()) info.update({display_name: data}) ctx = context.ctx() conductor.cluster_update(ctx, cluster_context.cluster, {'info': info}) def configure_general_environment(self, cluster_context, instances=None): LOG.debug('Executing post configure hooks') mapr_user_pass = pu.get_mapr_password(cluster_context.cluster) if not instances: instances = cluster_context.get_instances() def set_user_password(instance): LOG.debug('Setting password for user "mapr"') if self.mapr_user_exists(instance): with instance.remote() as r: r.execute_command( 'echo "%s:%s"|chpasswd' % ('mapr', mapr_user_pass), run_as_root=True) else: LOG.warning('User "mapr" does not exists') def create_home_mapr(instance): target_path = '/home/mapr' LOG.debug("Creating home directory for user 'mapr'") args = {'path': target_path, 'user': '******', 'group': 'mapr'} cmd = ('mkdir -p %(path)s && chown %(user)s:%(group)s %(path)s' % args) if self.mapr_user_exists(instance): with instance.remote() as r: r.execute_command(cmd, run_as_root=True) else: LOG.warning('User "mapr" does not exists') util.execute_on_instances(instances, set_user_password) util.execute_on_instances(instances, create_home_mapr) @el.provision_step(_("Execute configure.sh")) def _configure_sh_cluster(self, cluster_context, instances): LOG.debug('Executing configure.sh') if not instances: instances = cluster_context.get_instances() script = cluster_context.configure_sh db_specs = dict(mysql.MySQL.METRICS_SPECS._asdict()) db_specs.update({ 'host': mysql.MySQL.get_db_instance(cluster_context).internal_ip, 'port': mysql.MySQL.MYSQL_SERVER_PORT, }) with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('configure-sh-%s' % instance.id, self._configure_sh_instance, cluster_context, instance, script, db_specs) LOG.debug('Executing configure.sh successfully completed') @el.provision_event(instance_reference=2) def _configure_sh_instance(self, cluster_context, instance, command, specs): if not self.mapr_user_exists(instance): command += ' --create-user' if cluster_context.check_for_process(instance, mng.METRICS): command += (' -d %(host)s:%(port)s -du %(user)s -dp %(password)s ' '-ds %(db_name)s') % specs with instance.remote() as r: r.execute_command('sudo -i ' + command, timeout=_CONFIGURE_SH_TIMEOUT) @el.provision_step(_("Configure SSH connection")) def _configure_ssh_connection(self, cluster_context, instances): @el.provision_event() def configure_ssh(instance): echo_param = 'echo "KeepAlive yes" >> ~/.ssh/config' echo_timeout = 'echo "ServerAliveInterval 60" >> ~/.ssh/config' with instance.remote() as r: r.execute_command(echo_param) r.execute_command(echo_timeout) util.execute_on_instances(instances, configure_ssh) def mapr_user_exists(self, instance): with instance.remote() as r: ec, __ = r.execute_command( "id -u %s" % 'mapr', run_as_root=True, raise_when_error=False) return ec == 0 def post_start(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() LOG.debug('Executing service post start hooks') for service in cluster_context.cluster_services: updated = cluster_context.filter_instances(instances, service=service) service.post_start(cluster_context, updated) LOG.info('Post start hooks successfully executed') @el.provision_step(_("Set cluster mode")) def _set_cluster_mode(self, cluster_context, instances): cluster_mode = cluster_context.cluster_mode if not cluster_mode: return command = "maprcli cluster mapreduce set -mode %s" % cluster_mode @el.provision_event() def set_cluster_mode(instance): return util.execute_command([instance], command, run_as='mapr') util.execute_on_instances(instances, set_cluster_mode) @el.provision_step(_("Install MapR repositories")) def _install_mapr_repo(self, cluster_context, instances): distro_name = cluster_context.distro.name @el.provision_event() def install_mapr_repos(instance): return util.run_script(instance, ADD_MAPR_REPO_SCRIPT, "root", distro_name, **cluster_context.mapr_repos) util.execute_on_instances(instances, install_mapr_repos) def _update_services(self, cluster_context, instances): for service in cluster_context.cluster_services: updated = cluster_context.filter_instances(instances, service=service) service.update(cluster_context, updated) def _restart_services(self, cluster_context): restart = cluster_context.should_be_restarted for service, instances in restart.items(): service.restart(util.unique_list(instances)) def _post_configure_sh(self, cluster_context, instances): LOG.debug('Executing post configure.sh hooks') for service in cluster_context.cluster_services: service.post_configure_sh(cluster_context, instances) LOG.info('Post configure.sh hooks successfully executed') def _post_configure_services(self, cluster_context, instances): for service in cluster_context.cluster_services: service.post_configure(cluster_context, instances)
def _j2_render(template, arg_dict): if template: return template.render(arg_dict) else: raise e.PluginsInvalidDataException( _('Template object must be defined'))
class MapRFS(s.Service): _CREATE_DISK_LIST = 'plugins/mapr/resources/create_disk_list_file.sh' _DISK_SETUP_CMD = '/opt/mapr/server/disksetup -F /tmp/disk.list' _DISK_SETUP_TIMEOUT = 600 ENABLE_MAPR_DB_NAME = 'Enable MapR-DB' HEAP_SIZE_PERCENT_NAME = 'MapR-FS heap size percent' ENABLE_MAPR_DB_CONFIG = p.Config( name=ENABLE_MAPR_DB_NAME, applicable_target='general', scope='cluster', config_type="bool", priority=1, default_value=True, description=_('Specifies that MapR-DB is in use.')) HEAP_SIZE_PERCENT = p.Config( name=HEAP_SIZE_PERCENT_NAME, applicable_target='MapRFS', scope='cluster', config_type="int", priority=1, default_value=8, description=_( 'Specifies heap size for MapR-FS in percents of maximum value.')) def __init__(self): super(MapRFS, self).__init__() self._ui_name = 'MapRFS' self._node_processes = [CLDB, FILE_SERVER, NFS] self._ui_info = [ ('Container Location Database (CLDB)', CLDB, { s.SERVICE_UI: 'http://%s:7221' }), ] self._validation_rules = [ vu.at_least(1, CLDB), vu.each_node_has(FILE_SERVER), vu.on_same_node(CLDB, FILE_SERVER), vu.has_volumes(), ] def service_dir(self, cluster_context): return def home_dir(self, cluster_context): return def conf_dir(self, cluster_context): return '%s/conf' % cluster_context.mapr_home def post_install(self, cluster_context, instances): LOG.debug('Initializing MapR FS') instances = instances or cluster_context.get_instances() file_servers = cluster_context.filter_instances(instances, FILE_SERVER) utils.add_provisioning_step(cluster_context.cluster.id, _("Initializing MapR-FS"), len(file_servers)) with context.PluginsThreadGroup() as tg: for instance in file_servers: tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance, instance) LOG.info('MapR FS successfully initialized') @el.provision_event(instance_reference=1) def _init_mfs_instance(self, instance): self._generate_disk_list_file(instance, self._CREATE_DISK_LIST) self._execute_disksetup(instance) def _generate_disk_list_file(self, instance, path_to_disk_setup_script): LOG.debug('Creating disk list file') g.run_script(instance, path_to_disk_setup_script, 'root', *instance.storage_paths()) def _execute_disksetup(self, instance): with instance.remote() as rmt: rmt.execute_command(self._DISK_SETUP_CMD, run_as_root=True, timeout=self._DISK_SETUP_TIMEOUT) def get_configs(self): return [MapRFS.ENABLE_MAPR_DB_CONFIG, MapRFS.HEAP_SIZE_PERCENT] def get_config_files(self, cluster_context, configs, instance=None): default_path = 'plugins/mapr/services/maprfs/resources/cldb.conf' cldb_conf = bcf.PropertiesFile("cldb.conf") cldb_conf.remote_path = self.conf_dir(cluster_context) if instance: cldb_conf.fetch(instance) cldb_conf.parse(utils.get_file_text(default_path, 'sahara_plugin_mapr')) cldb_conf.add_properties(self._get_cldb_conf_props(cluster_context)) warden_conf = bcf.PropertiesFile("warden.conf") warden_conf.remote_path = "/opt/mapr/conf/" if instance: warden_conf.fetch(instance) warden_conf.add_properties({ 'service.command.mfs.heapsize.percent': configs[self.HEAP_SIZE_PERCENT_NAME] }) return [cldb_conf, warden_conf] def _get_cldb_conf_props(self, cluster_context): zookeepers = cluster_context.get_zookeeper_nodes_ip_with_port() return {'cldb.zookeeper.servers': zookeepers}
class Hue(s.Service): THRIFT_VERSIONS = [5, 7] THRIFT_VERSION = p.Config(name="Thrift version", applicable_target="Hue", scope='cluster', config_type="dropdown", config_values=[(v, v) for v in THRIFT_VERSIONS], priority=1, description=_('Specifies thrift version.')) def __init__(self): super(Hue, self).__init__() self._name = 'hue' self._ui_name = 'Hue' self._node_processes = [HUE] self._ui_info = None self._validation_rules = [ vu.exactly(1, HUE), vu.on_same_node(HUE, httpfs.HTTP_FS), vu.on_same_node(HUE_LIVY, spark.SPARK_SLAVE), ] self._priority = 2 def get_ui_info(self, cluster_context): # Hue uses credentials of the administrative user (PAM auth) return [('HUE', HUE, { s.SERVICE_UI: 'http://%s:8888', 'Username': '******', 'Password': pu.get_mapr_password(cluster_context.cluster) })] def get_configs(self): return [Hue.THRIFT_VERSION] def conf_dir(self, cluster_context): return '%s/desktop/conf' % self.home_dir(cluster_context) def get_config_files(self, cluster_context, configs, instance=None): template = 'plugins/mapr/services/hue/resources/hue_%s.template' # hue.ini hue_ini = bcf.TemplateFile("hue.ini") hue_ini.remote_path = self.conf_dir(cluster_context) hue_ini.parse( utils.get_file_text(template % self.version, 'sahara_plugin_mapr')) hue_ini.add_properties(self._get_hue_ini_props(cluster_context)) hue_ini.add_property("thrift_version", configs[self.THRIFT_VERSION.name]) # # hue.sh hue_sh_template = 'plugins/mapr/services/hue/' \ 'resources/hue_sh_%s.template' hue_sh = bcf.TemplateFile("hue.sh") hue_sh.remote_path = self.home_dir(cluster_context) + '/bin' hue_sh.parse( utils.get_file_text(hue_sh_template % self.version, 'sahara_plugin_mapr')) hue_sh.add_property('hadoop_version', cluster_context.hadoop_version) hue_sh.mode = 777 hue_instances = cluster_context.get_instances(HUE) for instance in hue_instances: if instance not in cluster_context.changed_instances(): cluster_context.should_be_restarted[self] += [instance] return [hue_ini, hue_sh] def _get_hue_ini_props(self, cluster_context): db_instance = mysql.MySQL.get_db_instance(cluster_context) is_yarn = cluster_context.cluster_mode == 'yarn' hue_specs = mysql.MySQL.HUE_SPECS rdbms_specs = mysql.MySQL.RDBMS_SPECS result = { 'db_host': db_instance.internal_ip, 'hue_name': hue_specs.db_name, 'hue_user': hue_specs.user, 'hue_password': hue_specs.password, 'rdbms_name': rdbms_specs.db_name, 'rdbms_user': rdbms_specs.user, 'rdbms_password': rdbms_specs.password, 'resource_manager_uri': cluster_context.resource_manager_uri, 'yarn_mode': is_yarn, 'rm_host': cluster_context.get_instance_ip(yarn.RESOURCE_MANAGER), 'webhdfs_url': cluster_context.get_instance_ip(httpfs.HTTP_FS), 'jt_host': cluster_context.get_instance_ip(mr.JOB_TRACKER), 'oozie_host': cluster_context.get_instance_ip(oozie.OOZIE), 'sqoop_host': cluster_context.get_instance_ip(sqoop.SQOOP_2_SERVER), 'impala_host': cluster_context.get_instance_ip(impala.IMPALA_STATE_STORE), 'zk_hosts_with_port': cluster_context.get_zookeeper_nodes_ip_with_port(), 'secret_key': self._generate_secret_key() } hive_host = cluster_context.get_instance(hive.HIVE_SERVER_2) if hive_host: hive_service = cluster_context.get_service(hive.HIVE_SERVER_2) result.update({ 'hive_host': hive_host.internal_ip, 'hive_version': hive_service.version, 'hive_conf_dir': hive_service.conf_dir(cluster_context), }) hbase_host = cluster_context.get_instance(hbase.HBASE_THRIFT) if hbase_host: hbase_service = cluster_context.get_service(hbase.HBASE_THRIFT) result.update({ 'hbase_host': hbase_host.internal_ip, 'hbase_conf_dir': hbase_service.conf_dir(cluster_context), }) livy_host = cluster_context.get_instance(HUE_LIVY) if livy_host: result.update({'livy_host': livy_host.internal_ip}) sentry_host = cluster_context.get_instance(sentry.SENTRY) if sentry_host: ui_name = sentry.Sentry().ui_name sentry_version = cluster_context.get_chosen_service_version( ui_name) sentry_service = cluster_context. \ _find_service_instance(ui_name, sentry_version) result.update({ 'sentry_host': sentry_host.internal_ip, 'sentry_conf': sentry_service.conf_dir(cluster_context) }) return result def post_install(self, cluster_context, instances): hue_instance = cluster_context.get_instance(HUE) @el.provision_event(name=_("Migrating Hue database"), instance=hue_instance) def migrate_database(remote, cluster_context): hue_home = self.home_dir(cluster_context) cmd = '%(activate)s && %(syncdb)s && %(migrate)s' args = { 'activate': 'source %s/build/env/bin/activate' % hue_home, 'syncdb': '%s/build/env/bin/hue syncdb --noinput' % hue_home, 'migrate': '%s/build/env/bin/hue migrate' % hue_home, } remote.execute_command(cmd % args, run_as_root=True, timeout=600) def hue_syncdb_workround(remote): cmd = 'printf "/opt/mapr/lib\n$JAVA_HOME/jre/lib/amd64/server\n"' \ ' | tee /etc/ld.so.conf.d/mapr-hue.conf && ldconfig' remote.execute_command(cmd, run_as_root=True) def centos7_workaround(remote): cmd = 'ln -s /lib64/libsasl2.so.3.0.0 /lib64/libsasl2.so.2' \ ' && rpm -ivh --nodeps http://yum.mariadb.org/5.5.49/' \ 'rhel7-amd64/rpms/MariaDB-5.5.49-centos7-x86_64-compat.rpm' remote.execute_command(cmd, run_as_root=True, raise_when_error=False) with hue_instance.remote() as r: LOG.debug("Executing Hue database migration") # workaround for centos7 if cluster_context.distro_version.split('.')[0] == '7': centos7_workaround(r) # temporary workaround to prevent failure of db migrate on mapr 5.2 hue_syncdb_workround(r) migrate_database(r, cluster_context) self._copy_hive_configs(cluster_context, hue_instance) self._install_jt_plugin(cluster_context, hue_instance) self._set_service_dir_owner(cluster_context, instances) def _copy_hive_configs(self, cluster_context, hue_instance): hive_server = cluster_context.get_instance(hive.HIVE_SERVER_2) if not hive_server or hive_server == hue_instance: LOG.debug('No Hive Servers found. Skip') return hive_service = cluster_context.get_service(hive.HIVE_SERVER_2) hive_conf_dir = hive_service.conf_dir(cluster_context) g.copy(hive_conf_dir, hive_server, hive_conf_dir, hue_instance, 'root') def update(self, cluster_context, instances=None): if self._should_restart(cluster_context, instances): self.restart(instances) def post_start(self, cluster_context, instances): self.update(cluster_context, instances=instances) def _should_restart(self, cluster_context, instances): app_services = [ impala.Impala(), hive.Hive(), hbase.HBase(), sqoop.Sqoop2(), spark.SparkOnYarn(), ] instances = [ cluster_context.filter_instances(instances, service=service) for service in app_services ] return bool(g.unique_list(itertools.chain(*instances))) def jt_plugin_path(self, cluster_context): path = ('%(home)s/desktop/libs/hadoop/java-lib' '/hue-plugins-%(version)s-mapr.jar') args = { 'home': self.home_dir(cluster_context), 'version': self.version, } return path % args @el.provision_event(name="Install Hue Job Tracker plugin", instance_reference=2) def _install_jt_plugin(self, cluster_context, hue_instance): LOG.debug("Copying Hue JobTracker plugin") job_trackers = cluster_context.get_instances(mr.JOB_TRACKER) if not job_trackers: LOG.debug('No JobTrackers found. Skip') return jt_plugin_src = self.jt_plugin_path(cluster_context) jt_plugin_dest = cluster_context.hadoop_lib + '/jt_plugin.jar' for jt in job_trackers: g.copy(jt_plugin_src, hue_instance, jt_plugin_dest, jt, 'root') def _generate_secret_key(self, length=80): ascii_alphanum = string.ascii_letters + string.digits generator = random.SystemRandom() return ''.join(generator.choice(ascii_alphanum) for _ in range(length))
class MapRPlugin(p.ProvisioningPluginBase): title = 'MapR Hadoop Distribution' description = _('The MapR Distribution provides a full Hadoop stack that' ' includes the MapR File System (MapR-FS), MapReduce,' ' a complete Hadoop ecosystem, and the MapR Control System' ' user interface') def _get_handler(self, hadoop_version): return vhf.VersionHandlerFactory.get().get_handler(hadoop_version) def get_title(self): return MapRPlugin.title def get_description(self): return MapRPlugin.description def get_labels(self): return { 'plugin_labels': {'enabled': {'status': True}}, 'version_labels': { '5.2.0.mrv2': {'enabled': {'status': True}}, } } def get_versions(self): return vhf.VersionHandlerFactory.get().get_versions() def get_node_processes(self, hadoop_version): return self._get_handler(hadoop_version).get_node_processes() def get_configs(self, hadoop_version): return self._get_handler(hadoop_version).get_configs() def configure_cluster(self, cluster): self._get_handler(cluster.hadoop_version).configure_cluster(cluster) def start_cluster(self, cluster): self._get_handler(cluster.hadoop_version).start_cluster(cluster) def validate(self, cluster): self._get_handler(cluster.hadoop_version).validate(cluster) def validate_scaling(self, cluster, existing, additional): v_handler = self._get_handler(cluster.hadoop_version) v_handler.validate_scaling(cluster, existing, additional) def scale_cluster(self, cluster, instances): v_handler = self._get_handler(cluster.hadoop_version) v_handler.scale_cluster(cluster, instances) def decommission_nodes(self, cluster, instances): v_handler = self._get_handler(cluster.hadoop_version) v_handler.decommission_nodes(cluster, instances) def get_edp_engine(self, cluster, job_type): v_handler = self._get_handler(cluster.hadoop_version) return v_handler.get_edp_engine(cluster, job_type) def get_edp_job_types(self, versions=None): res = {} for vers in self.get_versions(): if not versions or vers in versions: vh = self._get_handler(vers) res[vers] = vh.get_edp_job_types() return res def get_edp_config_hints(self, job_type, version): v_handler = self._get_handler(version) return v_handler.get_edp_config_hints(job_type) def get_open_ports(self, node_group): v_handler = self._get_handler(node_group.cluster.hadoop_version) return v_handler.get_open_ports(node_group) def get_health_checks(self, cluster): v_handler = self._get_handler(cluster.hadoop_version) return v_handler.get_cluster_checks(cluster) def get_image_arguments(self, hadoop_version): return self._get_handler(hadoop_version).get_image_arguments() def pack_image(self, hadoop_version, remote, test_only=False, image_arguments=None): version = self._get_handler(hadoop_version) version.pack_image(hadoop_version, remote, test_only=test_only, image_arguments=image_arguments) def validate_images(self, cluster, test_only=False, image_arguments=None): self._get_handler(cluster.hadoop_version).validate_images( cluster, test_only=test_only, image_arguments=image_arguments)
class Oozie(s.Service): def __init__(self): super(Oozie, self).__init__() self._name = 'oozie' self._ui_name = 'Oozie' self._node_processes = [OOZIE] self._cluster_defaults = ['oozie-default.json'] self._validation_rules = [vu.exactly(1, OOZIE)] self._ui_info = [('Oozie', OOZIE, { s.SERVICE_UI: 'http://%s:11000/oozie' })] def libext_path(self): return '/opt/mapr/oozie/oozie-%s/oozie-server/lib/' % self.version def get_config_files(self, cluster_context, configs, instance=None): oozie_site = bcf.HadoopXML("oozie-site.xml") oozie_site.remote_path = self.conf_dir(cluster_context) if instance: oozie_site.fetch(instance) oozie_site.load_properties(configs) oozie_site.add_properties(self._get_oozie_site_props(cluster_context)) return [oozie_site] def _get_oozie_site_props(self, cluster_context): oozie_specs = mysql.MySQL.OOZIE_SPECS return { 'oozie.db.schema.name': oozie_specs.db_name, 'oozie.service.JPAService.create.db.schema': True, 'oozie.service.JPAService.jdbc.driver': mysql.MySQL.DRIVER_CLASS, 'oozie.service.JPAService.jdbc.url': self._get_jdbc_uri(cluster_context), 'oozie.service.JPAService.jdbc.username': oozie_specs.user, 'oozie.service.JPAService.jdbc.password': oozie_specs.password, 'oozie.service.HadoopAccessorService.hadoop.configurations': '*=%s' % cluster_context.hadoop_conf } def _get_jdbc_uri(self, cluster_context): jdbc_uri = ('jdbc:mysql://%(db_host)s:%(db_port)s/%(db_name)s?' 'createDatabaseIfNotExist=true') jdbc_args = { 'db_host': mysql.MySQL.get_db_instance(cluster_context).internal_ip, 'db_port': mysql.MySQL.MYSQL_SERVER_PORT, 'db_name': mysql.MySQL.OOZIE_SPECS.db_name, } return jdbc_uri % jdbc_args def install(self, cluster_context, instances): # oozie requires executed configure.sh pass def post_configure(self, cluster_context, instances): super(Oozie, self).install(cluster_context, instances) oozie_instances = cluster_context.filter_instances(instances, service=self) for instance in oozie_instances: with instance.remote() as r: r.execute_command( 'sudo cp ' '/opt/mapr/oozie/oozie-%s/conf/warden.oozie.conf ' '/opt/mapr/conf/conf.d' % self.version) def post_install(self, cluster_context, instances): oozie_inst = cluster_context.get_instance(OOZIE) oozie_service = cluster_context.get_service(OOZIE) if oozie_service: symlink_cmd = ('cp /opt/mapr/lib/mysql-connector-*.jar %s' % self.libext_path()) with oozie_inst.remote() as r: LOG.debug('Installing MySQL connector for Oozie') r.execute_command(symlink_cmd, run_as_root=True, raise_when_error=False) self._set_service_dir_owner(cluster_context, instances) def post_start(self, cluster_context, instances): instances = cluster_context.filter_instances(instances, OOZIE) self._rebuild(cluster_context, instances) @el.provision_event(instance_reference=1) @g.remote_command(1) def _rebuild_oozie_war(self, remote, cluster_context): cmd = 'cp -f /opt/mapr-repository/ext-2.2.zip ' \ '%(home)s/libext/ext-2.2.zip &&' \ ' %(home)s/bin/oozie-setup.sh -hadoop %(version)s' \ ' /opt/mapr/hadoop/hadoop-%(version)s' args = { 'home': self.home_dir(cluster_context), 'version': cluster_context.hadoop_version } remote.execute_command(cmd % args, run_as_root=True) def update(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() instances = cluster_context.filter_instances(instances, OOZIE) self._rebuild(cluster_context, instances) @el.provision_step(_("Rebuilt Oozie war")) def _rebuild(self, cluster_context, instances): OOZIE.stop(list(filter(OOZIE.is_started, instances))) g.execute_on_instances(instances, self._rebuild_oozie_war, cluster_context) OOZIE.start(instances) context.sleep(OOZIE_START_DELAY)