def install_yarn(*args):
"""
Calls ansible playbook for the installation of yarn and all
required dependencies. Also formats and starts yarn or cloudera hadoop distribution.
Takes positional arguments as args tuple.
args: token, hosts_list, master_ip, cluster_name, orka_image_uuid, ssh_file, replication_factor, dfs_blocksize
"""
list_of_hosts = args[1]
master_hostname = list_of_hosts[0]['fqdn'].split('.', 1)[0]
cluster_size = len(list_of_hosts)
cluster_id = args[3].rsplit('-', 1)[1]
# Create ansible_hosts file
try:
hosts_filename = create_ansible_hosts(args[3], list_of_hosts, args[2])
# Run Ansible playbook
ansible_create_cluster(hosts_filename, cluster_size, args[4], args[5],
args[0], args[6], args[7])
# Format and start Hadoop cluster
set_cluster_state(args[0],
cluster_id,
'Yarn Cluster is active',
status='Active',
master_IP=args[2])
ansible_manage_cluster(cluster_id, 'format')
ansible_manage_cluster(cluster_id, 'start')
except Exception, e:
msg = 'Error while running Ansible %s' % e
raise RuntimeError(msg, error_ansible_playbook)
def install_yarn(*args):
"""
Calls ansible playbook for the installation of yarn and all
required dependencies. Also formats and starts yarn or cloudera hadoop distribution.
Takes positional arguments as args tuple.
args: token, hosts_list, master_ip, cluster_name, orka_image_uuid, ssh_file, replication_factor, dfs_blocksize
"""
from okeanos_utils import set_cluster_state
list_of_hosts = args[1]
master_hostname = list_of_hosts[0]['fqdn'].split('.', 1)[0] # list_of_host[0]['fqdn'] is like this: snf-654916.vm.okeanos.grnet.gr and we only need the snf-654916 part
cluster_size = len(list_of_hosts)
cluster_id = args[3].rsplit('-', 1)[1] # get the cluster's id
# Create ansible_hosts file
try:
hosts_filename = create_ansible_hosts(args[3], list_of_hosts, args[2])
# Run Ansible playbook
ansible_create_cluster(hosts_filename, cluster_size, args[4], args[5], args[0], args[6], args[7], args[8])
# Format and start Hadoop cluster
set_cluster_state(args[0], cluster_id,
'YARN Cluster is active', status='Active',
master_IP=args[2])
ansible_manage_cluster(cluster_id, 'format')
ansible_manage_cluster(cluster_id, 'start')
except Exception, e:
msg = 'Error while running Ansible %s' % e
raise RuntimeError(msg, error_ansible_playbook)
def create_bare_cluster(self):
"""Creates a bare ~okeanos cluster."""
server_home_path = expanduser('~')
server_ssh_keys = join(server_home_path, ".ssh/id_rsa.pub")
pub_keys_path = ''
logging.log(SUMMARY, 'Authentication verified')
current_task.update_state(state="Authenticated")
flavor_master, flavor_slaves, image_id = self.check_user_resources()
# Create name of cluster with [orka] prefix
cluster_name = '%s%s%s' % ('[orka]', '-', self.opts['cluster_name'])
self.opts['cluster_name'] = cluster_name
# Update db with cluster status as pending
task_id = current_task.request.id
self.cluster_id = db_cluster_create(self.opts, task_id)
# Append the cluster_id in the cluster name to create a unique name
# used later for naming various files, e.g. ansible_hosts file and
# create_cluster_debug file.
self.cluster_name_postfix_id = '%s%s%s' % (self.opts['cluster_name'],
'-', self.cluster_id)
# Check if user chose ssh keys or not.
if self.opts['ssh_key_selection'] is None or self.opts[
'ssh_key_selection'] == 'no_ssh_key_selected':
self.ssh_file = 'no_ssh_key_selected'
else:
self.ssh_key_file(self.cluster_name_postfix_id)
pub_keys_path = self.ssh_file
try:
cluster = Cluster(self.cyclades, self.opts['cluster_name'],
flavor_master, flavor_slaves, image_id,
self.opts['cluster_size'], self.net_client,
self.auth, self.project_id)
set_cluster_state(self.opts['token'], self.cluster_id,
"Creating ~okeanos cluster (1/3)")
self.HOSTNAME_MASTER_IP, self.server_dict = \
cluster.create(server_ssh_keys, pub_keys_path, '')
sleep(15)
except Exception, e:
# If error in bare cluster, update cluster status as destroyed
set_cluster_state(self.opts['token'],
self.cluster_id,
'Error',
status='Failed',
error=str(e.args[0]))
os.system('rm ' + self.ssh_file)
raise
def create_bare_cluster(self):
"""Creates a bare ~okeanos cluster."""
server_home_path = expanduser('~')
server_ssh_keys = join(server_home_path, ".ssh/id_rsa.pub")
pub_keys_path = ''
logging.log(SUMMARY, ' Authentication verified')
current_task.update_state(state="Authenticated")
flavor_master, flavor_slaves, image_id = self.check_user_resources()
# Create name of cluster with [orka] prefix
cluster_name = '%s%s%s' % ('[orka]', '-', self.opts['cluster_name'])
self.opts['cluster_name'] = cluster_name
# Update db with cluster status as pending
task_id = current_task.request.id
self.cluster_id = db_cluster_create(self.opts, task_id)
# Append the cluster_id in the cluster name to create a unique name
# used later for naming various files, e.g. ansible_hosts file and
# create_cluster_debug file.
self.cluster_name_postfix_id = '%s%s%s' % (self.opts['cluster_name'], '-', self.cluster_id)
# Check if user chose ssh keys or not.
if self.opts['ssh_key_selection'] is None or self.opts['ssh_key_selection'] == 'no_ssh_key_selected':
self.ssh_file = 'no_ssh_key_selected'
else:
self.ssh_key_file(self.cluster_name_postfix_id)
pub_keys_path = self.ssh_file
try:
cluster = Cluster(self.cyclades, self.opts['cluster_name'],
flavor_master, flavor_slaves,
image_id, self.opts['cluster_size'],
self.net_client, self.auth, self.project_id)
set_cluster_state(self.opts['token'], self.cluster_id, " Creating ~okeanos cluster (1/3)")
self.HOSTNAME_MASTER_IP, self.server_dict = \
cluster.create(server_ssh_keys, pub_keys_path, '')
sleep(15)
except Exception:
# If error in bare cluster, update cluster status as destroyed
set_cluster_state(self.opts['token'], self.cluster_id, 'Error', status='Destroyed')
os.system('rm ' + self.ssh_file)
raise
# Get master VM root password
self.master_root_pass = self.server_dict[0]['adminPass']
# Return master node ip and server dict
return self.HOSTNAME_MASTER_IP, self.server_dict
self.master_root_pass = self.server_dict[0]['adminPass']
# Return master node ip and server dict
return self.HOSTNAME_MASTER_IP, self.server_dict
def create_yarn_cluster(self):
"""Create Yarn cluster"""
try:
current_task.update_state(state="Started")
self.HOSTNAME_MASTER_IP, self.server_dict = self.create_bare_cluster(
)
except Exception, e:
logging.error(str(e.args[0]))
raise
# Update cluster info with the master VM root password.
set_cluster_state(self.opts['token'],
self.cluster_id,
'Configuring YARN cluster node communication (2/3)',
password=self.master_root_pass)
try:
list_of_hosts = reroute_ssh_prep(self.server_dict,
self.HOSTNAME_MASTER_IP)
set_cluster_state(self.opts['token'], self.cluster_id,
'Installing and configuring YARN (3/3)')
install_yarn(self.opts['token'], list_of_hosts,
self.HOSTNAME_MASTER_IP, self.cluster_name_postfix_id,
self.orka_image_uuid, self.ssh_file,
self.opts['replication_factor'],
self.opts['dfs_blocksize'])
except Exception, e:
# Get master VM root password
self.master_root_pass = self.server_dict[0]['adminPass']
# Return master node ip and server dict
return self.HOSTNAME_MASTER_IP, self.server_dict
def create_yarn_cluster(self):
"""Create Yarn cluster"""
try:
current_task.update_state(state="Started")
self.HOSTNAME_MASTER_IP, self.server_dict = self.create_bare_cluster()
except Exception, e:
logging.error(str(e.args[0]))
raise
# Update cluster info with the master VM root password.
set_cluster_state(self.opts['token'], self.cluster_id,
'Configuring YARN cluster node communication (2/3)',
password=self.master_root_pass)
try:
list_of_hosts = reroute_ssh_prep(self.server_dict,
self.HOSTNAME_MASTER_IP)
set_cluster_state(self.opts['token'], self.cluster_id,
'Installing and configuring YARN (3/3)')
install_yarn(self.opts['token'], list_of_hosts, self.HOSTNAME_MASTER_IP,
self.cluster_name_postfix_id, self.orka_image_uuid, self.ssh_file, self.opts['replication_factor'], self.opts['dfs_blocksize'])
except Exception, e:
logging.error(str(e.args[0]))
logging.error('Created cluster and resources will be deleted')
# If error in Yarn cluster, update cluster status as destroyed