Example #1
0
    def get_cluster_services(self, node_group=None):
        node_processes = None

        if node_group:
            node_processes = node_group.node_processes
        else:
            node_processes = [np for ng in self.cluster.node_groups for np in ng.node_processes]
            node_processes = g.unique_list(node_processes)
        services = g.unique_list(node_processes, self.get_service)

        return services + [swift.Swift()]
Example #2
0
    def get_cluster_services(self, node_group=None):
        node_processes = None

        if node_group:
            node_processes = node_group.node_processes
        else:
            node_processes = [np for ng in self.cluster.node_groups
                              for np in ng.node_processes]
            node_processes = g.unique_list(node_processes)
        services = g.unique_list(node_processes, self.get_service)

        return services + [swift.Swift()]
Example #3
0
 def _get_packages(self, node_processes):
     result = []
     result += self.dependencies
     result += [(np.package, self.version)
                for np in node_processes
                if np != SPARK_HISTORY_SERVER]
     return g.unique_list(result)
Example #4
0
 def get_open_ports(self, node_group):
     result = []
     for service in self.get_services():
         for node_process in service.node_processes:
             if node_process.ui_name in node_group.node_processes:
                 result += node_process.open_ports
     return util.unique_list(result)
Example #5
0
 def get_configs(self):
     if not self._configs:
         configs = [c for s in self.get_services() for c in s.get_configs()]
         configs += self._get_version_configs()
         configs += self._get_repo_configs()
         self._configs = util.unique_list(configs)
     return self._configs
Example #6
0
 def _get_packages(self, node_processes):
     result = []
     result += self.dependencies
     result += [(np.package, self.version)
                for np in node_processes
                if np != SPARK_HISTORY_SERVER]
     return g.unique_list(result)
 def get_open_ports(self, node_group):
     result = []
     for service in self.get_services():
         for node_process in service.node_processes:
             if node_process.ui_name in node_group.node_processes:
                 result += node_process.open_ports
     return util.unique_list(result)
Example #8
0
 def _install_ssh_keys(self, cluster_context, instances):
     slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
     masters = cluster_context.filter_instances(instances, SPARK_MASTER)
     instances = g.unique_list(masters + slaves)
     private_key = cluster_context.cluster.management_private_key
     public_key = cluster_context.cluster.management_public_key
     g.execute_on_instances(
         instances, g.install_ssh_key, 'mapr', private_key, public_key)
     g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
Example #9
0
 def _install_ssh_keys(self, cluster_context, instances):
     slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
     masters = cluster_context.filter_instances(instances, SPARK_MASTER)
     instances = g.unique_list(masters + slaves)
     private_key = cluster_context.cluster.management_private_key
     public_key = cluster_context.cluster.management_public_key
     g.execute_on_instances(
         instances, g.install_ssh_key, 'mapr', private_key, public_key)
     g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
Example #10
0
 def _should_restart(self, c_context, instances):
     app_services = [
         impala.Impala(),
         hive.Hive(),
         hbase.HBase(),
         sqoop.Sqoop2(),
     ]
     instances = [c_context.filter_instances(instances, service=service)
                  for service in app_services]
     return bool(g.unique_list(itertools.chain(*instances)))
Example #11
0
    def _write_config_files(self, cluster_context, instances):
        LOG.debug('Writing config files')

        @el.provision_event()
        def write_config_files(instance, config_files):
            for file in config_files:
                util.write_file(instance, file.path, file.data, mode=file.mode,
                                owner="mapr")

        node_groups = util.unique_list(instances, lambda i: i.node_group)
        for node_group in node_groups:
            config_files = cluster_context.get_config_files(node_group)
            ng_instances = [i for i in node_group.instances if i in instances]
            util.execute_on_instances(ng_instances, write_config_files,
                                      config_files=config_files)

        LOG.debug("Config files are successfully written")
Example #12
0
    def _write_config_files(self, cluster_context, instances):
        LOG.debug('Writing config files')

        @el.provision_event()
        def write_config_files(instance, config_files):
            for path, data in six.iteritems(config_files):
                util.mkdir(instance, os.path.dirname(path), owner="root")
                util.write_file(instance, path, data, owner="root")

        node_groups = util.unique_list(instances, lambda i: i.node_group)
        for node_group in node_groups:
            config_files = cluster_context.get_config_files(node_group)
            ng_instances = [i for i in node_group.instances if i in instances]
            util.execute_on_instances(ng_instances, write_config_files,
                                      config_files=config_files)

        LOG.debug("Config files are successfully written")
    def _write_config_files(self, cluster_context, instances):
        LOG.debug('Writing config files')

        @el.provision_event()
        def write_config_files(instance, config_files):
            for file in config_files:
                util.write_file(instance,
                                file.path,
                                file.data,
                                mode=file.mode,
                                owner="mapr")

        node_groups = util.unique_list(instances, lambda i: i.node_group)
        for node_group in node_groups:
            config_files = cluster_context.get_config_files(node_group)
            ng_instances = [i for i in node_group.instances if i in instances]
            util.execute_on_instances(ng_instances,
                                      write_config_files,
                                      config_files=config_files)

        LOG.debug("Config files are successfully written")
 def _restart_services(self, cluster_context):
     restart = cluster_context.should_be_restarted
     for service, instances in six.iteritems(restart):
         service.restart(util.unique_list(instances))
 def _restart_services(self, cluster_context):
     restart = cluster_context.should_be_restarted
     for service, instances in six.iteritems(restart):
         service.restart(util.unique_list(instances))
 def get_node_groups(instances):
     return util.unique_list(instances, lambda i: i.node_group)
Example #17
0
 def get_node_groups(instances):
     return util.unique_list(instances, lambda i: i.node_group)
 def get_configs(self):
     if not self._configs:
         configs = [c for s in self.get_services() for c in s.get_configs()]
         configs += self._get_version_configs()
         self._configs = util.unique_list(configs)
     return self._configs