def configure_os(self, instances): # instances non-empty u.add_provisioning_step( instances[0].cluster_id, _("Configure OS"), len(instances)) with context.PluginsThreadGroup() as tg: for inst in instances: tg.spawn('cdh-repo-conf-%s' % inst.instance_name, self._configure_repo_from_inst, inst)
def disable_repos(cluster): if configs.use_base_repos_needed(cluster): LOG.debug("Using base repos") return instances = plugin_utils.get_instances(cluster) with context.PluginsThreadGroup() as tg: for inst in instances: tg.spawn("disable-repos-%s" % inst.instance_name, _disable_repos_on_inst, inst)
def _setup_agents(instances, manager_address): plugin_utils.add_provisioning_step(instances[0].cluster.id, _("Set up Ambari agents"), len(instances)) with context.PluginsThreadGroup() as tg: for inst in instances: tg.spawn("hwx-agent-setup-%s" % inst.id, _setup_agent, inst, manager_address) LOG.debug("Ambari agents have been installed")
def install_packages(self, instances, packages): # instances non-empty u.add_provisioning_step( instances[0].cluster_id, _("Install packages"), len(instances)) with context.PluginsThreadGroup() as tg: for i in instances: tg.spawn('cdh-inst-pkgs-%s' % i.instance_name, self._install_pkgs, i, packages)
def update_configs(self, instances): # instances non-empty utils.add_provisioning_step( instances[0].cluster_id, _("Update configs"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn("update-configs-%s" % instance.instance_name, self._update_configs, instance) context.sleep(1)
def start_zk_server(instances): utils.add_provisioning_step(instances[0].cluster_id, utils.start_process_event_message("ZooKeeper"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-start-processes-%s' % instance.instance_name, _start_zk_processes, instance, 'start')
def start_cloudera_agents(self, instances): # instances non-empty u.add_provisioning_step( instances[0].cluster_id, _("Start Cloudera Agents"), len(instances)) with context.PluginsThreadGroup() as tg: for i in instances: tg.spawn('cdh-agent-start-%s' % i.instance_name, self._start_cloudera_agent, i)
def _start_slave_processes(self, sl_instances): if len(sl_instances) == 0: return utils.add_provisioning_step(sl_instances[0].cluster_id, utils.start_process_event_message("Slave"), len(sl_instances)) with context.PluginsThreadGroup() as tg: for i in sl_instances: tg.spawn('storm-start-sl-%s' % i.instance_name, self._start_slaves, i)
def _start_zookeeper_processes(self, zk_instances): if len(zk_instances) == 0: return utils.add_provisioning_step( zk_instances[0].cluster_id, utils.start_process_event_message("Zookeeper"), len(zk_instances)) with context.PluginsThreadGroup() as tg: for i in zk_instances: tg.spawn('storm-start-zk-%s' % i.instance_name, self._start_zookeeper, i)
def _start_datanode_processes(self, dn_instances): if len(dn_instances) == 0: return utils.add_provisioning_step( dn_instances[0].cluster_id, utils.start_process_event_message("DataNodes"), len(dn_instances)) with context.PluginsThreadGroup() as tg: for i in dn_instances: tg.spawn('spark-start-dn-%s' % i.instance_name, self._start_datanode, i)
def configure_swift(self, cluster, instances=None): if self.c_helper.is_swift_enabled(cluster): if not instances: instances = u.get_instances(cluster) u.add_provisioning_step( cluster.id, _("Configure Swift"), len(instances)) with context.PluginsThreadGroup() as tg: for i in instances: tg.spawn('cdh-swift-conf-%s' % i.instance_name, self._configure_swift_to_inst, i) swift_helper.install_ssl_certs(instances)
def post_install(self, cluster_context, instances): LOG.debug('Initializing MapR FS') instances = instances or cluster_context.get_instances() file_servers = cluster_context.filter_instances(instances, FILE_SERVER) utils.add_provisioning_step(cluster_context.cluster.id, _("Initializing MapR-FS"), len(file_servers)) with context.PluginsThreadGroup() as tg: for instance in file_servers: tg.spawn('init-mfs-%s' % instance.id, self._init_mfs_instance, instance) LOG.info('MapR FS successfully initialized')
def _push_configs_to_nodes(self, cluster, extra, new_instances): all_instances = utils.get_instances(cluster) utils.add_provisioning_step(cluster.id, _("Push configs to nodes"), len(all_instances)) with context.PluginsThreadGroup() as tg: for instance in all_instances: if instance in new_instances: tg.spawn('storm-configure-%s' % instance.instance_name, self._push_configs_to_new_node, cluster, extra, instance) else: tg.spawn('storm-reconfigure-%s' % instance.instance_name, self._push_configs_to_existing_node, cluster, extra, instance)
def refresh_zk_servers(cluster, to_delete_instances=None): instances = vu.get_zk_servers(cluster) if to_delete_instances: for instance in to_delete_instances: if instance in instances: instances.remove(instance) utils.add_provisioning_step(cluster.id, utils.start_process_event_message("ZooKeeper"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-restart-processes-%s' % instance.instance_name, _start_zk_processes, instance, 'restart')
def start_dn_nm_processes(instances): filternames = ['datanode', 'nodemanager'] instances = utils.instances_with_services(instances, filternames) if len(instances) == 0: return utils.add_provisioning_step( instances[0].cluster_id, utils.start_process_event_message("DataNodes, NodeManagers"), len(instances)) with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): processes = set(instance.node_group.node_processes) processes = processes.intersection(filternames) tg.spawn('vanilla-start-processes-%s' % instance.instance_name, _start_processes, instance, list(processes))
def _configure_sh_cluster(self, cluster_context, instances): LOG.debug('Executing configure.sh') if not instances: instances = cluster_context.get_instances() script = cluster_context.configure_sh db_specs = dict(mysql.MySQL.METRICS_SPECS._asdict()) db_specs.update({ 'host': mysql.MySQL.get_db_instance(cluster_context).internal_ip, 'port': mysql.MySQL.MYSQL_SERVER_PORT, }) with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('configure-sh-%s' % instance.id, self._configure_sh_instance, cluster_context, instance, script, db_specs) LOG.debug('Executing configure.sh successfully completed')
def provision_keypairs(cluster, instances=None): extra = cluster.extra.to_dict() if cluster.extra else {} # use same keypair for scaling keypair = extra.get('vanilla_keypair') if not instances: instances = utils.get_instances(cluster) else: # scaling if not keypair: # cluster created before mitaka, skipping provisioning return if not keypair: private, public = utils.generate_key_pair() keypair = {'public': public, 'private': private} extra['vanilla_keypair'] = keypair extra['vanilla_keypair']['private'] = _store_secret(keypair['private']) conductor.cluster_update(context.ctx(), cluster, {'extra': extra}) else: keypair['private'] = _get_secret(keypair['private']) with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('provision-key-%s' % instance.instance_name, _provision_key, instance, keypair)
def _stop_nodes(self, instances, sys_service): with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn('stop-%s-%s' % (sys_service, instance.id), self._stop_service, instance, sys_service)
def execute_on_instances(instances, function, *args, **kwargs): with context.PluginsThreadGroup() as tg: for instance in instances: t_name = '%s-execution' % function.__name__ tg.spawn(t_name, function, instance, *args, **kwargs)