def configure_general_environment(self, cluster_context, instances=None):
LOG.debug('Executing post configure hooks')
if not instances:
instances = cluster_context.get_instances()
def set_user_password(instance):
LOG.debug('Setting password for user "mapr"')
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(
'echo "%s:%s"|chpasswd' % ('mapr', 'mapr'),
run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
def create_home_mapr(instance):
target_path = '/home/mapr'
LOG.debug("Creating home directory for user 'mapr'")
args = {'path': target_path}
cmd = 'mkdir -p %(path)s && chown mapr:mapr %(path)s' % args
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
util.execute_on_instances(instances, set_user_password)
util.execute_on_instances(instances, create_home_mapr)
def configure_general_environment(self, cluster_context, instances=None):
LOG.debug('Executing post configure hooks')
mapr_user_pass = pu.get_mapr_password(cluster_context.cluster)
if not instances:
instances = cluster_context.get_instances()
def set_user_password(instance):
LOG.debug('Setting password for user "%s"' % pu.MAPR_USER_NAME)
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(
'echo "%s:%s"|chpasswd' %
(pu.MAPR_USER_NAME, mapr_user_pass),
run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
def create_home_mapr(instance):
target_path = '/home/%s' % pu.MAPR_USER_NAME
LOG.debug("Creating home directory for user '%s'" %
pu.MAPR_USER_NAME)
args = {'path': target_path,
'user': pu.MAPR_USER_NAME,
'group': _MAPR_GROUP_NAME}
cmd = ('mkdir -p %(path)s && chown %(user)s:%(group)s %(path)s'
% args)
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
util.execute_on_instances(instances, set_user_password)
util.execute_on_instances(instances, create_home_mapr)
def configure_general_environment(self, cluster_context, instances=None):
LOG.debug('Executing post configure hooks')
mapr_user_pass = pu.get_mapr_password(cluster_context.cluster)
if not instances:
instances = cluster_context.get_instances()
def set_user_password(instance):
LOG.debug('Setting password for user "mapr"')
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command('echo "%s:%s"|chpasswd' %
('mapr', mapr_user_pass),
run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
def create_home_mapr(instance):
target_path = '/home/mapr'
LOG.debug("Creating home directory for user 'mapr'")
args = {'path': target_path, 'user': '******', 'group': 'mapr'}
cmd = ('mkdir -p %(path)s && chown %(user)s:%(group)s %(path)s' %
args)
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
util.execute_on_instances(instances, set_user_password)
util.execute_on_instances(instances, create_home_mapr)
def _install_security_repos(self, cluster_context, instances):
LOG.debug("Installing security repos")
@el.provision_event()
def install_security_repos(instance):
return util.run_script(instance, ADD_SECURITY_REPO_SCRIPT, "root")
util.execute_on_instances(instances, install_security_repos)
def _install_security_repos(self, cluster_context, instances):
LOG.debug("Installing security repos")
@el.provision_event()
def install_security_repos(instance):
return util.run_script(instance, ADD_SECURITY_REPO_SCRIPT, "root")
util.execute_on_instances(instances, install_security_repos)
def _install_ssh_keys(self, cluster_context, instances):
slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
masters = cluster_context.filter_instances(instances, SPARK_MASTER)
instances = g.unique_list(masters + slaves)
private_key = cluster_context.cluster.management_private_key
public_key = cluster_context.cluster.management_public_key
g.execute_on_instances(
instances, g.install_ssh_key, 'mapr', private_key, public_key)
g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
def _install_ssh_keys(self, cluster_context, instances):
slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
masters = cluster_context.filter_instances(instances, SPARK_MASTER)
instances = g.unique_list(masters + slaves)
private_key = cluster_context.cluster.management_private_key
public_key = cluster_context.cluster.management_public_key
g.execute_on_instances(
instances, g.install_ssh_key, 'mapr', private_key, public_key)
g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
def _install_mapr_repo(self, cluster_context, instances):
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mapr_repos(instance):
return util.run_script(instance, ADD_MAPR_REPO_SCRIPT, "root",
distro_name, **cluster_context.mapr_repos)
util.execute_on_instances(instances, install_mapr_repos)
def _configure_ssh_connection(self, cluster_context, instances):
def keep_alive_connection(instance):
echo_param = 'echo "KeepAlive yes" >> ~/.ssh/config'
echo_timeout = 'echo "ServerAliveInterval 60" >> ~/.ssh/config'
with instance.remote() as r:
r.execute_command(echo_param)
r.execute_command(echo_timeout)
util.execute_on_instances(instances, keep_alive_connection)
def _configure_ssh_connection(self, cluster_context, instances):
def keep_alive_connection(instance):
echo_param = 'echo "KeepAlive yes" >> ~/.ssh/config'
echo_timeout = 'echo "ServerAliveInterval 60" >> ~/.ssh/config'
with instance.remote() as r:
r.execute_command(echo_param)
r.execute_command(echo_timeout)
util.execute_on_instances(instances, keep_alive_connection)
def _install_mapr_repo(self, cluster_context, instances):
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mapr_repos(instance):
return util.run_script(instance, ADD_MAPR_REPO_SCRIPT, "root",
distro_name, **cluster_context.mapr_repos)
util.execute_on_instances(instances, install_mapr_repos)
def _install_java(self, cluster_context, instances):
LOG.debug("Installing Java")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_java(instance):
return util.run_script(instance, INSTALL_JAVA_SCRIPT, "root",
distro_name)
util.execute_on_instances(instances, install_java)
def _install_mysql_client(self, cluster_context, instances):
LOG.debug("Installing MySQL client")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mysql_client(instance):
return util.run_script(instance, INSTALL_MYSQL_CLIENT, "root",
distro_name)
util.execute_on_instances(instances, install_mysql_client)
def _install_mysql_client(self, cluster_context, instances):
LOG.debug("Installing MySQL client")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_mysql_client(instance):
return util.run_script(instance, INSTALL_MYSQL_CLIENT,
"root", distro_name)
util.execute_on_instances(instances, install_mysql_client)
def _install_java(self, cluster_context, instances):
LOG.debug("Installing Java")
distro_name = cluster_context.distro.name
@el.provision_event()
def install_java(instance):
return util.run_script(instance, INSTALL_JAVA_SCRIPT,
"root", distro_name)
util.execute_on_instances(instances, install_java)
def _set_cluster_mode(self, cluster_context, instances):
cluster_mode = cluster_context.cluster_mode
if not cluster_mode:
return
command = "maprcli cluster mapreduce set -mode %s" % cluster_mode
@el.provision_event()
def set_cluster_mode(instance):
return util.execute_command([instance], command, run_as="mapr")
util.execute_on_instances(instances, set_cluster_mode)
def _set_cluster_mode(self, cluster_context, instances):
cluster_mode = cluster_context.cluster_mode
if not cluster_mode:
return
command = "maprcli cluster mapreduce set -mode %s" % cluster_mode
@el.provision_event()
def set_cluster_mode(instance):
return util.execute_command([instance], command, run_as="mapr")
util.execute_on_instances(instances, set_cluster_mode)
def _wait_for_status(self, instances, status, sleep=3, timeout=60):
def poll_status(instance):
operation_name = _('Wait for {node_process} on {instance}'
' to change status to "{status}"')
args = {
'node_process': self.ui_name,
'instance': instance.instance_name,
'status': status.name,
}
return polls.poll(
get_status=lambda: self.status(instance) == status,
operation_name=operation_name.format(**args),
timeout=timeout,
sleep=sleep,
)
util.execute_on_instances(instances, poll_status)
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for file in config_files:
util.write_file(instance, file.path, file.data, mode=file.mode,
owner="mapr")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances, write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for path, data in six.iteritems(config_files):
util.mkdir(instance, os.path.dirname(path), owner="root")
util.write_file(instance, path, data, owner="root")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances, write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _wait_for_status(self, instances, status, sleep=3, timeout=60):
def poll_status(instance):
operation_name = _('Wait for {node_process} on {instance}'
' to change status to "{status}"')
args = {
'node_process': self.ui_name,
'instance': instance.instance_name,
'status': status.name,
}
return polls.poll(
get_status=lambda: self.status(instance) == status,
operation_name=operation_name.format(**args),
timeout=timeout,
sleep=sleep,
)
util.execute_on_instances(instances, poll_status)
def _configure_topology(self, context, instances):
def write_file(instance, path, data):
with instance.remote() as r:
r.write_file_to(path, data, run_as_root=True)
LOG.debug("Configuring cluster topology")
is_node_aware = context.is_node_aware
if is_node_aware:
topo = th.generate_topology_map(context.cluster, is_node_aware)
topo = "\n".join(["%s %s" % i for i in six.iteritems(topo)]) + "\n"
data_path = "%s/topology.data" % context.mapr_home
script = files.get_file_text(_TOPO_SCRIPT)
script_path = "%s/topology.sh" % context.mapr_home
util.execute_on_instances(instances, write_file, data_path, topo)
util.execute_on_instances(instances, util.write_file, script_path, script, "+x", "root")
else:
LOG.debug("Data locality is disabled.")
LOG.info(_LI("Cluster topology successfully configured"))
def _configure_topology(self, context, instances):
def write_file(instance, path, data):
with instance.remote() as r:
r.write_file_to(path, data, run_as_root=True)
LOG.debug('Configuring cluster topology')
is_node_aware = context.is_node_aware
if is_node_aware:
topo = th.generate_topology_map(context.cluster, is_node_aware)
topo = '\n'.join(['%s %s' % i for i in six.iteritems(topo)]) + '\n'
data_path = '%s/topology.data' % context.mapr_home
script = files.get_file_text(_TOPO_SCRIPT)
script_path = '%s/topology.sh' % context.mapr_home
util.execute_on_instances(instances, write_file, data_path, topo)
util.execute_on_instances(
instances, util.write_file, script_path, script, '+x', 'root')
else:
LOG.debug('Data locality is disabled.')
LOG.info(_LI('Cluster topology successfully configured'))
def _configure_topology(self, context, instances):
def write_file(instance, path, data):
with instance.remote() as r:
r.write_file_to(path, data, run_as_root=True)
LOG.debug('Configuring cluster topology')
is_node_aware = context.is_node_aware
if is_node_aware:
topo = th.generate_topology_map(context.cluster, is_node_aware)
topo = '\n'.join(['%s %s' % i for i in six.iteritems(topo)]) + '\n'
data_path = '%s/topology.data' % context.mapr_home
script = files.get_file_text(_TOPO_SCRIPT)
script_path = '%s/topology.sh' % context.mapr_home
util.execute_on_instances(instances, write_file, data_path, topo)
util.execute_on_instances(
instances, util.write_file, script_path, script, '+x', 'root')
else:
LOG.debug('Data locality is disabled.')
LOG.info(_LI('Cluster topology successfully configured'))
def _configure_topology(self, context, instances):
LOG.debug("Configuring cluster topology")
topology_map = context.topology_map
topology_map = ("%s %s" % item for item in six.iteritems(topology_map))
topology_map = "\n".join(topology_map) + "\n"
data_path = "%s/topology.data" % context.mapr_home
script = files.get_file_text(_TOPO_SCRIPT)
script_path = '%s/topology.sh' % context.mapr_home
@el.provision_event()
def write_topology_data(instance):
util.write_file(instance, data_path, topology_map, owner="root")
util.write_file(instance, script_path, script,
mode="+x", owner="root")
util.execute_on_instances(instances, write_topology_data)
LOG.info(_LI('Cluster topology successfully configured'))
def _configure_topology(self, context, instances):
LOG.debug("Configuring cluster topology")
topology_map = context.topology_map
topology_map = ("%s %s" % item for item in six.iteritems(topology_map))
topology_map = "\n".join(topology_map) + "\n"
data_path = "%s/topology.data" % context.mapr_home
script = files.get_file_text(_TOPO_SCRIPT)
script_path = '%s/topology.sh' % context.mapr_home
@el.provision_event()
def write_topology_data(instance):
util.write_file(instance, data_path, topology_map, owner="root")
util.write_file(instance, script_path, script,
mode="+x", owner="root")
util.execute_on_instances(instances, write_topology_data)
LOG.info(_LI('Cluster topology successfully configured'))
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for file in config_files:
util.write_file(instance,
file.path,
file.data,
mode=file.mode,
owner="mapr")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances,
write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def configure_general_environment(self, cluster_context, instances=None):
LOG.debug('Executing post configure hooks')
if not instances:
instances = cluster_context.get_instances()
def create_user(instance):
return util.run_script(instance, ADD_MAPR_USER, "root")
def set_user_password(instance):
LOG.debug('Setting password for user "mapr"')
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command('echo "%s:%s"|chpasswd' %
('mapr', 'mapr'),
run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
def create_home_mapr(instance):
target_path = '/home/mapr'
LOG.debug("Creating home directory for user 'mapr'")
args = {'path': target_path}
cmd = 'mkdir -p %(path)s && chown mapr:mapr %(path)s' % args
if self.mapr_user_exists(instance):
with instance.remote() as r:
r.execute_command(cmd, run_as_root=True)
else:
LOG.warning(_LW('User "mapr" does not exists'))
util.execute_on_instances(instances, create_user)
util.execute_on_instances(instances, set_user_password)
util.execute_on_instances(instances, create_home_mapr)
def _prepare_bare_image(self, cluster_context, instances):
LOG.debug("Preparing bare image")
if d.UBUNTU == cluster_context.distro:
LOG.debug("Installing security repos")
util.execute_on_instances(instances, util.run_script, ADD_SECURITY_REPO_SCRIPT, "root")
d_name = cluster_context.distro.name
LOG.debug("Installing Java")
util.execute_on_instances(instances, util.run_script, INSTALL_JAVA_SCRIPT, "root", d_name)
LOG.debug("Installing Scala")
util.execute_on_instances(instances, util.run_script, INSTALL_SCALA_SCRIPT, "root", d_name)
LOG.debug("Installing MySQL client")
util.execute_on_instances(instances, util.run_script, INSTALL_MYSQL_CLIENT, "root", d_name)
LOG.debug("Bare images successfully prepared")
def _prepare_bare_image(self, cluster_context, instances):
LOG.debug('Preparing bare image')
d_name = cluster_context.distro.name
LOG.debug('Installing Java')
util.execute_on_instances(
instances, util.run_script, INSTALL_JAVA_SCRIPT, 'root', d_name)
LOG.debug('Installing Scala')
util.execute_on_instances(
instances, util.run_script, INSTALL_SCALA_SCRIPT, 'root', d_name)
LOG.debug('Installing MySQL client')
util.execute_on_instances(
instances, util.run_script, INSTALL_MYSQL_CLIENT, 'root', d_name)
LOG.debug('Bare images successfully prepared')
def _prepare_bare_image(self, cluster_context, instances):
LOG.debug('Preparing bare image')
d_name = cluster_context.distro.name
LOG.debug('Installing Java')
util.execute_on_instances(
instances, util.run_script, INSTALL_JAVA_SCRIPT, 'root', d_name)
LOG.debug('Installing Scala')
util.execute_on_instances(
instances, util.run_script, INSTALL_SCALA_SCRIPT, 'root', d_name)
LOG.debug('Installing MySQL client')
util.execute_on_instances(
instances, util.run_script, INSTALL_MYSQL_CLIENT, 'root', d_name)
LOG.debug('Bare images successfully prepared')
def _install(_context, _instances):
g.execute_on_instances(_instances,
self._install_packages_on_instance,
_context)
def _install_ui(self, cluster_context, instances):
OOZIE.stop(filter(OOZIE.is_started, instances))
g.execute_on_instances(
instances, self._rebuild_oozie_war, cluster_context)
OOZIE.start(instances)
def _rebuild(self, cluster_context, instances):
OOZIE.stop(filter(OOZIE.is_started, instances))
g.execute_on_instances(
instances, self._rebuild_oozie_war, cluster_context)
OOZIE.start(instances)
con.sleep(OOZIE_START_DELAY)
def _install(_context, _instances):
g.execute_on_instances(_instances,
self._install_packages_on_instance,
_context)
def install(self, cluster_context, instances):
g.execute_on_instances(instances, self._install_packages_on_instance,
cluster_context)
def _rebuild(self, cluster_context, instances):
OOZIE.stop(filter(OOZIE.is_started, instances))
g.execute_on_instances(instances, self._rebuild_oozie_war,
cluster_context)
OOZIE.start(instances)
context.sleep(OOZIE_START_DELAY)
def _install_mapr_repo(self, cluster_context, instances):
d_name = cluster_context.distro.name
util.execute_on_instances(
instances, util.run_script, ADD_MAPR_REPO_SCRIPT, 'root', d_name,
**cluster_context.mapr_repos)
def _install_ui(self, cluster_context, instances):
OOZIE.stop(filter(OOZIE.is_started, instances))
g.execute_on_instances(
instances, self._rebuild_oozie_war, cluster_context)
OOZIE.start(instances)
def _install_mapr_repo(self, cluster_context, instances):
d_name = cluster_context.distro.name
util.execute_on_instances(
instances, util.run_script, ADD_MAPR_REPO_SCRIPT, 'root', d_name,
**cluster_context.mapr_repos)