def step_2_init_infra(create_wait=0):
init_start_ts = _dt.utcnow()
log.info("------------- Getting Environment at [%s]", init_start_ts)
horton.cbd = infra.get_cloudbreak(purge=horton.global_purge,
create_wait=create_wait)
log.info("------------- Connecting to Environment")
public_ip = horton.cbd.public_ips[0]
cbd_url = 'https://' + public_ip + '/cb/api'
cad_url = 'https://' + public_ip + ':7189'
log.info("Setting Cloudbreak endpoint to %s", cbd_url)
utils.set_endpoint(cbd_url)
log.info("Setting Altus Director endpoint to %s", cad_url)
utils.set_endpoint(cad_url)
log.info("------------- Authenticating to Cloudbreak")
cbd_auth_success = security.service_login(
service='cloudbreak',
username=config.profile['email'],
password=security.get_secret('ADMINPASSWORD'),
bool_response=False)
if not cbd_auth_success:
raise ConnectionError("Couldn't login to Cloudbreak")
else:
log.info('Logged into Cloudbreak at [%s]', cbd_url)
log.info("------------- Authenticating to Altus Director")
cad_auth_success = security.service_login(
service='director',
username=config.profile['username'],
password=security.get_secret('ADMINPASSWORD'),
bool_response=False)
if not cad_auth_success:
raise ConnectionError("Couldn't login to Director")
else:
log.info('Logged into Director at [%s]', cad_url)
# Cloudbreak may have just booted and not be ready for queries yet
log.info("Waiting for Cloudbreak API Calls to be available")
utils.wait_to_complete(deploy.list_blueprints,
bool_response=True,
whoville_delay=5,
whoville_max_wait=120)
# Director may not be ready for queries yet
log.info("Waiting for Altus Director API Calls to be available")
utils.wait_to_complete(director.list_environments,
bool_response=True,
whoville_delay=5,
whoville_max_wait=120)
log.info("------------- Setting Deployment Credential")
horton.cred = deploy.get_credential(config.profile['namespace'] +
'credential',
create=True,
purge=horton.global_purge)
init_finish_ts = _dt.utcnow()
diff_ts = init_finish_ts - init_start_ts
log.info("Completed Infrastructure Init at [%s] after [%d] seconds",
init_finish_ts, diff_ts.seconds)
def validate_profile():
log.info("Validating provided profile.yml")
horton = Horton()
# Check Profile is imported
if not config.profile:
raise ValueError("whoville Config Profile is not populated with"
"deployment controls, cannot proceed")
# Check Profile version
if 'profilever' not in config.profile:
raise ValueError("Your Profile is out of date, please recreate your "
"Profile from the template")
if config.profile['profilever'] < config.min_profile_ver:
raise ValueError("Your Profile is out of date, please recreate your "
"Profile from the template. Profile v3 requires an ssh private key or pem file.")
# Handle SSH
if 'sshkey_file' in config.profile and config.profile['sshkey_file']:
assert config.profile['sshkey_file'].endswith('.pem')
from Crypto.PublicKey import RSA
pem_key = RSA.importKey(fs_read(config.profile['sshkey_file']))
config.profile['sshkey_pub'] = pem_key.publickey().exportKey().decode()
config.profile['sshkey_priv'] = pem_key.exportKey().decode()
config.profile['sshkey_name'] = os.path.basename(config.profile['sshkey_file']).split('.')[0]
else:
assert any(k in config.profile for k in ['ssh_key_priv', 'sshkey_priv'])
assert all(k in config.profile for k in ['sshkey_pub', 'sshkey_name'])
# Check Namespace
assert isinstance(horton.namespace, six.string_types),\
"Namespace must be string"
assert len(horton.namespace) >= 2,\
"Namespace must be at least 2 characters"
# Check Password
if 'password' in config.profile and config.profile['password']:
horton.cache['ADMINPASSWORD'] = config.profile['password']
else:
horton.cache['ADMINPASSWORD'] = security.get_secret('ADMINPASSWORD')
password_test = re.compile(r'^(?=.*[A-Za-z])(?=.*\d)[A-Za-z\d-]{12,}$')
if not bool(password_test.match(horton.cache['ADMINPASSWORD'])):
raise ValueError("Password doesn't match Platform spec."
"Requires 12+ characters, at least 1 letter and "
"number, may also contain -")
# Check Provider
provider = config.profile.get('platform')['provider']
assert provider in ['EC2', 'AZURE_ARM', 'GCE']
# TODO: Read in the profile template, check it has all matching keys
# Check Profile Namespace is validate
ns_test = re.compile(r'[a-z0-9-]')
if not bool(ns_test.match(horton.namespace)):
raise ValueError("Namespace must only contain 0-9 a-z -")
# Check storage bucket matches expected format
if 'bucket' in config.profile:
if provider == 'EC2':
bucket_test = re.compile(r'[a-z0-9.-]')
elif provider == 'AZURE_ARM':
bucket_test = re.compile(r'[a-z0-9@]')
elif provider == 'GCE':
bucket_test = re.compile(r'[a-z0-9.-]')
else:
raise ValueError("Platform Provider not supported")
if not bool(bucket_test.match(config.profile['bucket'])):
raise ValueError("Bucket name doesn't match Platform spec")
def create_deployment(cm_ver,
env_name=None,
tem_name=None,
dep_name=None,
tls_start=False,
csds=None):
assert isinstance(cm_ver, six.string_types)
env_name = env_name if env_name else horton.cdcred.name
log.info("Using Environment [%s]", env_name)
tem_name = tem_name if tem_name else dep_name if dep_name else horton.cdcred.name
log.info("Using Virtual Template [%s]", tem_name)
dep_name = dep_name if dep_name else env_name + '-' + str(cm_ver).replace(
'.', '-')
template = get_instance_template(tem_name=tem_name)
if not template:
log.info("Template [%s] not found, creating from defaults", tem_name)
create_instance_template(tem_name=tem_name)
while not template:
sleep(2)
template = get_instance_template(tem_name=tem_name)
if cm_ver[0] == '6':
repo = 'https://archive.cloudera.com/cm6/' + cm_ver + '/redhat7/yum/'
repo_key = 'https://archive.cloudera.com/cm6/' + cm_ver +\
'/redhat7/yum/RPM-GPG-KEY-cloudera'
elif cm_ver[0] == '5':
repo = 'https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/' +\
cm_ver + '/'
repo_key = 'https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/' \
'RPM-GPG-KEY-cloudera'
else:
raise ValueError("Only CM 5 or 6 supported")
try:
log.info("Attempting to create Deployment [%s]", dep_name)
return cd.DeploymentsApi(horton.cad).create(
environment=env_name,
deployment_template=cd.DeploymentTemplate(
name=dep_name,
manager_virtual_instance=cd.VirtualInstance(id=str(
uuid.uuid4()),
template=template),
password=security.get_secret('ADMINPASSWORD'),
enable_enterprise_trial=True,
repository_key_url=repo_key,
repository=repo,
tls_enabled=tls_start,
csds=csds,
java_installation_strategy='NONE'))
except ApiException as e:
if e.status == 409:
log.warning('A deployment with the same name already exists')
else:
raise e
def create_cluster(cluster_def,
dep_name,
workers=3,
env_name=None,
scripts=None):
env_name = env_name if env_name else horton.cdcred.name
cdh_ver = str(cluster_def['products']['CDH'])
services = cluster_def['services']
cluster_name = cluster_def['name']
products = cluster_def['products']
if 'post_create_scripts' in cluster_def and cluster_def[
'post_create_scripts'] is not None:
log.info("Including post_create_scripts in cluster of %s",
cluster_def['post_create_scripts'])
post_create_scripts = []
for script_name in cluster_def['post_create_scripts']:
log.info("Adding script %s like [%s]", script_name,
scripts[script_name][:50])
post_create_scripts.append(cd.Script(content=scripts[script_name]))
else:
post_create_scripts = None
if cdh_ver[0] == '5':
load_parcels = [
'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/'
]
elif cdh_ver[0] == '6':
load_parcels = [
'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/'
]
else:
raise ValueError("Only CDH versions 5 or 6 supported")
if 'parcels' in cluster_def:
load_parcels += cluster_def['parcels']
# Default Role and Service configs
services_configs = {}
master_setups = {}
master_configs = {}
worker_setups = {}
worker_configs = {}
if 'HDFS' in services:
master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE']
worker_setups['HDFS'] = ['DATANODE']
if 'YARN' in services:
master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY']
worker_setups['YARN'] = ['NODEMANAGER']
if 'ZOOKEEPER' in services:
master_setups['ZOOKEEPER'] = ['SERVER']
if 'HBASE' in services:
master_setups['HBASE'] = ['MASTER']
worker_setups['HBASE'] = ['REGIONSERVER']
if 'HIVE' in services:
master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE']
if 'HUE' in services:
master_setups['HUE'] = ['HUE_SERVER']
if 'KUDU' in services:
master_setups['KUDU'] = ['KUDU_MASTER']
worker_setups['KUDU'] = ['KUDU_TSERVER']
master_configs['KUDU'] = {
'KUDU_MASTER': {
'fs_wal_dir': "/data0/kudu/masterwal",
'fs_data_dirs': "/data1/kudu/master"
}
}
worker_configs['KUDU'] = {
'KUDU_TSERVER': {
'fs_wal_dir': "/data0/kudu/tabletwal",
'fs_data_dirs': "/data1/kudu/tablet"
}
}
if 'IMPALA' in services:
master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE']
worker_setups['IMPALA'] = ['IMPALAD']
if 'NIFI' in services:
worker_setups['NIFI'] = ['NIFI_NODE']
if 'NIFIREGISTRY' in services:
master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER']
if 'NIFITOOLKITCA' in services:
master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER']
services_configs['NIFITOOLKITCA'] = {
'nifi.toolkit.tls.ca.server.token':
security.get_secret('MASTERKEY')
}
if 'KAFKA' in services:
worker_setups['KAFKA'] = ['KAFKA_BROKER']
services_configs['KAFKA'] = {'producer.metrics.enable': True}
if 'SCHEMAREGISTRY' in services:
master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER']
# Handle Services Configs overrides
if 'servicesconfigs' in cluster_def.keys():
for k, v in cluster_def['servicesconfigs']:
services_configs[k] = v
# Handle virtual instance generation
master_vi = [
create_virtual_instance(tem_name=dep_name + '-' + cluster_name +
'-master',
scripts=[
'''sudo -i
yum install mysql mariadb-server epel-release -y # MariaDB
yum -y install npm gcc-c++ make # SMM-UI
npm install forever -g # SMM-UI
systemctl enable mariadb
service mariadb start
mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION"
mysql --execute="CREATE DATABASE streamsmsgmgr DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'streamsmsgmgr'@'localhost' IDENTIFIED BY 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' identified by 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' WITH GRANT OPTION"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' identified by 'streamsmsgmgr'"
mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' WITH GRANT OPTION"
mysql --execute="FLUSH PRIVILEGES"
mysql --execute="COMMIT"'''
])
]
worker_vi = [
create_virtual_instance(tem_name=dep_name + '-' + cluster_name +
'-worker') for _ in range(0, workers)
]
try:
cd.ClustersApi(horton.cad).create(
environment=env_name,
deployment=dep_name,
cluster_template=cd.ClusterTemplate(
name=cluster_name,
product_versions=products,
parcel_repositories=load_parcels,
services=services,
services_configs=services_configs,
virtual_instance_groups={
'masters':
cd.VirtualInstanceGroup(
name='masters',
min_count=1,
service_type_to_role_types=master_setups,
role_types_configs=master_configs,
virtual_instances=master_vi),
'workers':
cd.VirtualInstanceGroup(
name='workers',
min_count=workers,
service_type_to_role_types=worker_setups,
role_types_configs=worker_configs,
virtual_instances=worker_vi)
},
post_create_scripts=post_create_scripts))
except ApiException as e:
if e.status == 409:
log.error("Cluster %s already exists", cluster_name)
raise ValueError("Cluster %s already exists", cluster_name)
else:
raise e
def init_cbreak_infra(create=True, create_wait=0):
init_start_ts = _dt.utcnow()
log.info("------------- Getting Environment at [%s]", init_start_ts)
horton.cbd = infra.get_cloudbreak(purge=horton.global_purge,
create_wait=create_wait,
create=create)
if not horton.cbd:
if create:
# Create has failed, throw error
raise ValueError(
"Cloudbreak Create requested but failed, exiting...")
else:
return None
else:
log.info("Found existing Cloudbreak in Namespace, connecting...")
log.info("------------- Connecting to Environment")
if horton.cbd.public_ips:
public_ip = horton.cbd.public_ips[0]
else:
public_ip = horton.cbd.name + config.profile['platform']['domain']
cbd_url = 'https://' + public_ip + '/cb/api'
cad_url = 'https://' + public_ip + ':7189'
log.info("Setting Cloudbreak endpoint to %s", cbd_url)
utils.set_endpoint(cbd_url)
log.info("Setting Altus Director endpoint to %s", cad_url)
utils.set_endpoint(cad_url)
log.info("------------- Authenticating to Cloudbreak")
cbd_auth_success = security.service_login(
service='cloudbreak',
username=config.profile['email'],
password=security.get_secret('ADMINPASSWORD'),
bool_response=False)
if not cbd_auth_success:
raise ConnectionError("Couldn't login to Cloudbreak")
else:
log.info('Logged into Cloudbreak at [%s]', cbd_url)
log.info("------------- Authenticating to Altus Director")
cad_auth_success = security.service_login(
service='director',
username=config.profile['username'],
password=security.get_secret('ADMINPASSWORD'),
bool_response=False)
if not cad_auth_success:
raise ConnectionError("Couldn't login to Director")
else:
log.info('Logged into Director at [%s]', cad_url)
# Cloudbreak may have just booted and not be ready for queries yet
log.info("Waiting for Cloudbreak API Calls to be available")
utils.wait_to_complete(deploy.list_blueprints,
bool_response=True,
whoville_delay=5,
whoville_max_wait=120)
# Director may not be ready for queries yet
log.info("Waiting for Altus Director API Calls to be available")
utils.wait_to_complete(director.list_environments,
bool_response=True,
whoville_delay=5,
whoville_max_wait=120)
# Validating Cloudbreak version
if not deploy.check_cloudbreak_version():
raise ValueError(
"Cloudbreak server is older than configured minimum version of %s",
str(config.cb_ver))
# Creating Environment Credentials
log.info("------------- Setting Deployment Credential")
log.info("Ensuring Credential for Cloudbreak")
horton.cbcred = deploy.get_credential(config.profile['namespace'] +
'credential',
create=True,
purge=horton.global_purge)
if ((config.profile['platform']['provider'] == 'EC2')
or (config.profile['platform']['provider'] == 'GCE')):
log.info("Ensuring Environment Credential for Director")
horton.cadcred = director.get_environment()
init_finish_ts = _dt.utcnow()
diff_ts = init_finish_ts - init_start_ts
log.info("Completed Infrastructure Init at [%s] after [%d] seconds",
init_finish_ts, diff_ts.seconds)
def create_cloudbreak(session, cbd_name):
public_ip = requests.get('https://ipv4.icanhazip.com').text.rstrip()
net_rules = [{
'protocol': 'tcp',
'from_port': 9443,
'to_port': 9443,
'cidr_ips': ['0.0.0.0/0']
}, {
'protocol': -1,
'from_port': 1,
'to_port': 65535,
'cidr_ips': [public_ip + '/32']
}, {
'protocol': 'tcp',
'from_port': 443,
'to_port': 443,
'cidr_ips': ['0.0.0.0/0']
}, {
'protocol': 'tcp',
'from_port': 22,
'to_port': 22,
'cidr_ips': ['0.0.0.0/0']
}]
if session.type == 'ec2':
s_boto3 = create_boto3_session()
aws_clean_cloudformation(s_boto3)
log.info("Selecting OS Image for Cloudbreak")
images = list_images(session,
filters={
'name': '*CentOS Linux 7 x86_64 HVM EBS ENA*',
})
image = sorted(images, key=lambda k: k.extra['description'][-7:])
if not image:
raise ValueError("Couldn't find a valid Centos7 Image")
else:
image = image[-1]
bd = image.extra['block_device_mapping'][0]
root_vol = {
'VirtualName': None,
'DeviceName': bd['device_name'],
'Ebs': {
'VolumeSize': 50,
'VolumeType': bd['ebs']['volume_type'],
'DeleteOnTermination': True
}
}
log.info("Fetching list of suitable machine types")
machines = list_sizes_aws(session,
cpu_min=4,
cpu_max=4,
mem_min=16000,
mem_max=20000)
if not machines:
raise ValueError("Couldn't find a VM of the right size")
else:
machine = machines[-1]
log.info("Fetching list of available networks")
networks = list_networks(session)
network = sorted(networks, key=lambda k: k.extra['is_default'])
if not network:
raise ValueError("There should be at least one network, this "
"is rather unexpected")
else:
network = network[-1]
log.info("Fetching subnets in Network")
subnets = list_subnets(session, {'extra.vpc_id': network.id})
subnets = sorted(subnets, key=lambda k: k.state)
ec2_resource = s_boto3.resource('ec2')
if not subnets:
raise ValueError("Expecting at least one subnet on a network")
subnet = [
x for x in subnets
if ec2_resource.Subnet(x.id).map_public_ip_on_launch
]
if not subnet:
raise ValueError("There are no subnets with auto provisioning of "
"public IPs enabled..."
"enable public IP auto provisioning on at least "
"one subnet in the default VPC")
else:
subnet = subnet[0]
log.info("Fetching Security groups matching namespace")
sec_group = list_security_groups(session, {'name': namespace})
if not sec_group:
log.info("Namespace Security group not found, creating")
_ = session.ex_create_security_group(
name=namespace + 'whoville-default',
description=namespace + 'whoville-default Security Group',
vpc_id=network.id)
sec_group = list_security_groups(session, {'name': namespace})[-1]
else:
sec_group = sec_group[-1]
net_rules.append({
'protocol': -1,
'group_pairs': [{
'group_id': sec_group.id
}],
'from_port': 0,
'to_port': 0
})
# security group loopback doesn't work well on AWS, need to use subnet
net_rules.append({
'protocol': -1,
'cidr_ips': [subnet.extra['cidr_block']],
'from_port': 0,
'to_port': 0
})
for rule in net_rules:
add_sec_rule_to_ec2_group(session, rule, sec_group.id)
log.info("Checking for expected SSH Keypair")
ssh_key = list_keypairs(session,
{'name': config.profile['sshkey_name']})
if not ssh_key:
ssh_key = session.import_key_pair_from_string(
name=config.profile['sshkey_name'],
key_material=config.profile['sshkey_pub'])
else:
ssh_key = [
x for x in ssh_key if x.name == config.profile['sshkey_name']
][0]
log.info("Creating Static IP for Cloudbreak")
try:
static_ips = [
x for x in session.ex_describe_all_addresses()
if x.instance_id is None
]
except InvalidCredsError:
static_ips = None
if not static_ips:
static_ip = session.ex_allocate_address()
else:
static_ip = static_ips[0]
if not static_ip:
raise ValueError("Couldn't get a Static IP for Cloudbreak")
# This is just a tidy way of specifying a script
cb_ver = config.profile.get('cloudbreak_ver')
cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver
script_lines = [
"#!/bin/bash", "cd /root", "export cb_ver=" + cb_ver,
"export uaa_secret=" + security.get_secret('MASTERKEY'),
"export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'),
"export uaa_default_email=" + config.profile['email'],
"export public_ip=" + static_ip.ip,
"source <(curl -sSL https://raw.githubusercontent.com/Chaffelson"
"/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)"
]
script = '\n'.join(script_lines)
cbd = create_node(session=session,
name=cbd_name,
image=image,
machine=machine,
params={
'ex_security_group_ids': [sec_group.id],
'ex_subnet': subnet,
'ex_assign_public_ip': True,
'ex_blockdevicemappings': [root_vol],
'ex_keyname': ssh_key.name,
'ex_userdata': script
})
# inserting hard wait to bypass race condition where returned node ID
# is not actually available to the list API call yet
sleep(5)
log.info("Waiting for Cloudbreak Instance to be Available...")
session.wait_until_running(nodes=[cbd])
log.info("Cloudbreak Infra Booted at [%s]", cbd)
log.info("Assigning Static IP to Cloudbreak")
session.ex_associate_address_with_node(cbd, static_ip)
# Assign Role ARN
if 'infraarn' in config.profile['platform']:
log.info("Found infraarn in Profile, associating with Cloudbreak")
infra_arn = config.profile['platform']['infraarn']
client = s_boto3.client('ec2')
client.associate_iam_instance_profile(IamInstanceProfile={
'Arn':
infra_arn,
'Name':
infra_arn.rsplit('/')[-1]
},
InstanceId=cbd.id)
# get updated node information
cbd = list_nodes(session, {'name': cbd_name})
cbd = [x for x in cbd if x.state == 'running']
if cbd:
return cbd[0]
else:
raise ValueError("Failed to create new Cloubreak Instance")
if session.type == 'azure_arm':
ssh_key = config.profile['sshkey_pub']
resource_group = namespace + 'cloudbreak-group'
network_name = namespace + 'cloudbreak-network'
subnet_name = namespace + 'cloudbreak-subnet'
sec_group_name = namespace + 'cloudbreak-secgroup'
public_ip_name = namespace + 'cloudbreak-ip'
nic_name = namespace + 'cloudbreak-nic'
disk_account_name = namespace + 'diskaccount'
disk_account_name = disk_account_name.replace('-', '')
# ToDo: examine cleaning Azure Resource Groups
log.info("Creating Resource Group...")
token = get_azure_token()
azure_resource_client = create_azure_session(token, 'resource')
azure_resource_client.resource_groups.create_or_update(
resource_group,
{'location': config.profile.get('platform')['region']})
image = session.list_images(ex_publisher='OpenLogic',
ex_offer='CentOS-CI',
ex_sku='7-CI')
if not image:
raise ValueError("Couldn't find a valid Centos7 Image")
else:
image = image[-1]
machines = list_sizes_azure(session,
cpu_min=4,
cpu_max=4,
mem_min=16384,
mem_max=20480)
if not machines:
raise ValueError("Couldn't find a VM of the right size")
else:
machine = machines[0]
log.info("Checking for disk storage account, please wait...")
azure_storage_client = create_azure_session(token, 'storage')
try:
azure_storage_client.storage_accounts.create(
resource_group, disk_account_name, {
'location': config.profile.get('platform')['region'],
'sku': {
'name': 'standard_lrs'
},
'kind': 'StorageV2'
}).wait()
except Exception:
log.info("Found existing os disk account...")
log.info("Looking for existing network resources...")
azure_network_client = create_azure_session(token, 'network')
try:
log.info("Getting Vnet...")
network = azure_network_client.virtual_networks.get(
resource_group, network_name)
log.info("Getting Network Interface...")
nic = azure_network_client.network_interfaces.get(
resource_group, nic_name)
log.info("Getting Public IP...")
public_ip = azure_network_client.public_ip_addresses.get(
resource_group, public_ip_name)
except Exception:
log.info("No Vnet exists for this namepsace, creating...")
network = azure_network_client.virtual_networks.create_or_update(
resource_group, network_name, {
'location': config.profile.get('platform')['region'],
'address_space': {
'address_prefixes': ['10.0.0.0/16']
}
})
network = network.result()
log.info("Creating Subnet...")
subnet = azure_network_client.subnets.create_or_update(
resource_group, network_name, subnet_name,
{'address_prefix': '10.0.0.0/24'})
log.info("Creating Public IP...")
public_ip = azure_network_client.public_ip_addresses.create_or_update(
resource_group, public_ip_name, {
'location': config.profile.get('platform')['region'],
'public_ip_allocation_method': 'static'
})
subnet = subnet.result()
public_ip = public_ip.result()
log.info("Creating Security Group...")
sec_group = azure_network_client.network_security_groups.create_or_update(
resource_group, sec_group_name, {
'location':
config.profile.get('platform')['region'],
'security_rules': [{
'name': 'ssh_rule',
'description': 'Allow SSH',
'protocol': 'Tcp',
'source_port_range': '*',
'destination_port_range': '22',
'source_address_prefix': 'Internet',
'destination_address_prefix': '*',
'access': 'Allow',
'priority': 100,
'direction': 'Inbound'
}, {
'name': 'http_rule',
'description': 'Allow HTTP',
'protocol': 'Tcp',
'sourcePortRange': '*',
'destinationPortRange': '80',
'sourceAddressPrefix': 'Internet',
'destinationAddressPrefix': '*',
'access': 'Allow',
'priority': 101,
'direction': 'Inbound'
}, {
'name': 'https_rule',
'provisioningState': 'Succeeded',
'description': 'Allow HTTPS',
'protocol': 'Tcp',
'sourcePortRange': '*',
'destinationPortRange': '443',
'sourceAddressPrefix': 'Internet',
'destinationAddressPrefix': '*',
'access': 'Allow',
'priority': 102,
'direction': 'Inbound'
}, {
'name': 'knox_https_rule',
'provisioningState': 'Succeeded',
'description': 'Allow CB HTTPS',
'protocol': 'Tcp',
'sourcePortRange': '*',
'destinationPortRange': '8443',
'sourceAddressPrefix': 'Internet',
'destinationAddressPrefix': '*',
'access': 'Allow',
'priority': 103,
'direction': 'Inbound'
}, {
'name': 'cb_https_rule',
'provisioningState': 'Succeeded',
'description': 'Allow CB HTTPS',
'protocol': 'Tcp',
'sourcePortRange': '*',
'destinationPortRange': '9443',
'sourceAddressPrefix': 'Internet',
'destinationAddressPrefix': '*',
'access': 'Allow',
'priority': 104,
'direction': 'Inbound'
}, {
'name': 'altus_http_rule',
'provisioningState': 'Succeeded',
'description': 'Allow Altus HTTP',
'protocol': 'Tcp',
'sourcePortRange': '*',
'destinationPortRange': '7189',
'sourceAddressPrefix': 'Internet',
'destinationAddressPrefix': '*',
'access': 'Allow',
'priority': 105,
'direction': 'Inbound'
}]
})
sec_group = sec_group.result()
log.info("Creating Network Interface...")
nic = azure_network_client.network_interfaces.create_or_update(
resource_group, nic_name, {
'location':
config.profile.get('platform')['region'],
'network_security_group': {
'id': sec_group.id
},
'ip_configurations': [{
'name': 'default',
'subnet': {
'id': subnet.id
},
'public_ip_address': {
'id': public_ip.id
}
}]
})
nic = nic.result()
# End network exception handling for missing vnet
public_ip = public_ip.ip_address
cb_ver = config.profile.get('cloudbreak_ver')
cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver
script_lines = [
"#!/bin/bash", "cd /root", "yum install -y wget",
"wget -O jq https://github.com/stedolan/jq/releases/download/"
"jq-1.5/jq-linux64", "chmod +x ./jq", "cp jq /usr/bin",
"export cb_ver=" + cb_ver,
"export uaa_secret=" + security.get_secret('MASTERKEY'),
"export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'),
"export uaa_default_email=" + config.profile['email'],
"export public_ip=" + public_ip,
"source <(curl -sSL https://raw.githubusercontent.com/Chaffelson"
"/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)"
]
script = '\n'.join(script_lines)
script = script.encode()
script = str(base64.urlsafe_b64encode(script))\
.replace("b'","").replace("'","")
log.info("Creating Virtual Machine...")
log.info("with custom_data string like: " + script[:100])
azure_compute_client = create_azure_session(token, 'compute')
cbd = azure_compute_client.virtual_machines.create_or_update(
resource_group, cbd_name, {
'location': config.profile.get('platform')['region'],
'os_profile': {
'computer_name': cbd_name,
'admin_username': '******',
'linux_configuration': {
'disable_password_authentication': True,
'ssh': {
'public_keys': [{
'path':
'/home/{}/.ssh/authorized_keys'.format(
'centos'),
'key_data':
ssh_key
}]
}
},
'custom_data': script
},
'hardware_profile': {
'vm_size': 'Standard_DS3_v2'
},
'storage_profile': {
'image_reference': {
'publisher': 'Redhat',
'offer': 'RHEL',
'sku': '7-RAW-CI',
'version': 'latest'
},
'os_disk': {
'name': cbd_name,
'create_option': 'fromImage',
'vhd': {
'uri':
'https://{}.blob.core.windows.net/'
'vhds/{}.vhd'.format(disk_account_name, cbd_name)
}
},
},
'network_profile': {
'network_interfaces': [{
'id': nic.id,
'primary': True
}]
}
})
log.info("Waiting for Cloudbreak Instance to be Available...")
cbd.wait()
cbd = list_nodes(session, {'name': cbd_name})
cbd = [x for x in cbd if x.state == 'running']
if cbd:
return cbd[0]
else:
raise ValueError("Failed to create new Cloubreak Instance")
elif session.type == 'gce':
region = config.profile['platform']['region']
cbd_name = namespace + 'cloudbreak'
public_ip_name = namespace + 'cloudbreak-public-ip'
subnet_name = namespace + 'cloudbreak-subnet'
firewall_name = namespace + 'cloudbreak-secgroup'
ssh_key = config.profile['sshkey_pub']
log.info("Looking for existing network...")
networks = session.ex_list_networks()
network = [x for x in networks if x.mode == 'auto']
if not network:
raise ValueError("There should be at least one network")
else:
network = network[-1]
log.info("Found network: " + network.name)
log.info("Looking for existing subnets...")
subnets = session.ex_list_subnetworks(region=region)
subnet = [x for x in subnets if x.name == 'default']
if not subnet:
session.ex_create_subnetwork(name=subnet_name,
region=region,
network=network)
else:
subnet = subnet[-1]
subnet_name = subnet.name
log.info("Found existing subnet called: " + subnet_name)
log.info("Getting Public IP...")
try:
public_ip = session.ex_get_address(name=public_ip_name,
region=region)
log.info("Found existing Public IP matching name: " +
public_ip_name)
except ResourceNotFoundError:
public_ip = session.ex_create_address(name=public_ip_name,
region=region)
log.info("Creating new Public IP with name: " + public_ip_name)
images = session.list_images()
image = [
x for x in images
if x.extra['family'] == 'centos-7' and 'centos-7' in x.name
]
zones = session.ex_list_zones()
zone = [x for x in zones if region in x.name and x.status == 'UP']
if not zone:
raise ValueError("Couldn't find a zone for the requested region..")
else:
zone = zone[-1]
if not image:
raise ValueError("Couldn't find a valid Centos7 Image")
else:
image = image[-1]
machines = list_sizes_gce(session,
location=region,
cpu_min=4,
cpu_max=4,
mem_min=13000,
mem_max=20000)
if not machines:
raise ValueError("Couldn't find a VM of the right size")
else:
machine = machines[-1]
log.info("Creating Security Group...")
try:
_ = session.ex_get_firewall(name=firewall_name)
log.info("Found existing firewall definition called: " +
firewall_name)
except ResourceNotFoundError:
log.info("Creating new firewall definition called: " +
firewall_name)
net_rules = [{
'IPProtocol': 'tcp',
'ports': ['22', '443', '8443', '9443', '7189']
}]
_ = session.ex_create_firewall(name=firewall_name,
network=network,
allowed=net_rules,
target_tags=[cbd_name])
cb_ver = config.profile.get('cloudbreak_ver')
cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver
script_lines = [
"#!/bin/bash", "cd /root", "export cb_ver=" + cb_ver,
"export uaa_secret=" + security.get_secret('MASTERKEY'),
"export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'),
"export uaa_default_email=" + config.profile['email'],
"export public_ip=" + public_ip.address,
"source <(curl -sSL https://raw.githubusercontent.com/Chaffelson"
"/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)"
]
script = '\n'.join(script_lines)
metadata = {
'items': [{
'key': 'startup-script',
'value': script
}, {
'key': 'ssh-keys',
'value': 'centos:' + ssh_key
}]
}
log.info("Creating Cloudbreak instance...")
cbd = session.create_node(name=cbd_name,
size=machine,
image=image,
location=zone,
ex_network=network,
external_ip=public_ip,
ex_metadata=metadata,
ex_tags=[cbd_name])
log.info("Waiting for Cloudbreak Instance to be Available...")
session.wait_until_running(nodes=[cbd])
cbd = list_nodes(session, {'name': cbd_name})
cbd = [x for x in cbd if x.state == 'running']
if cbd:
return cbd[0]
else:
raise ValueError("Failed to create new Cloubreak Instance")
else:
raise ValueError("Cloudbreak AutoDeploy only supported on EC2, Azure, "
"and GCE")
def create_cluster(cdh_ver,
workers=3,
services=None,
env_name=None,
dep_name=None,
clus_name=None,
parcels=None):
env_name = env_name if env_name else horton.cdcred.name
dep_name = dep_name if dep_name else env_name + '-' + cdh_ver.replace(
'.', '-')
services = services if services else ['HDFS', 'YARN']
if cdh_ver[0] == '5':
load_parcels = [
'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/'
]
elif cdh_ver[0] == '6':
load_parcels = [
'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/'
]
else:
raise ValueError("Only CDH versions 5 or 6 supported")
if parcels:
load_parcels += parcels
if cdh_ver[0] == '6':
# CDH6 already has Kafka
load_parcels = [x for x in load_parcels if 'kafka' not in x]
products = {'CDH': cdh_ver}
if 'NIFI' in str(services):
products['CFM'] = '1'
if 'KAFKA' in str(services) and cdh_ver[0] == '5':
products['KAFKA'] = '4'
if 'SCHEMAREGISTRY' in str(services):
products['SCHEMAREGISTRY'] = '0.7'
services_configs = {}
master_setups = {}
master_configs = {}
worker_setups = {}
worker_configs = {}
if 'HDFS' in services:
master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE']
worker_setups['HDFS'] = ['DATANODE']
if 'YARN' in services:
master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY']
worker_setups['YARN'] = ['NODEMANAGER']
if 'ZOOKEEPER' in services:
master_setups['ZOOKEEPER'] = ['SERVER']
if 'HBASE' in services:
master_setups['HBASE'] = ['MASTER']
worker_setups['HBASE'] = ['REGIONSERVER']
if 'HIVE' in services:
master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE']
if 'HUE' in services:
master_setups['HUE'] = ['HUE_SERVER']
if 'KUDU' in services:
master_setups['KUDU'] = ['KUDU_MASTER']
worker_setups['KUDU'] = ['KUDU_TSERVER']
master_configs['KUDU'] = {
'KUDU_MASTER': {
'fs_wal_dir': "/data0/kudu/masterwal",
'fs_data_dirs': "/data1/kudu/master"
}
}
worker_configs['KUDU'] = {
'KUDU_TSERVER': {
'fs_wal_dir': "/data0/kudu/tabletwal",
'fs_data_dirs': "/data1/kudu/tablet"
}
}
if 'IMPALA' in services:
master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE']
worker_setups['IMPALA'] = ['IMPALAD']
if 'NIFI' in services:
worker_setups['NIFI'] = ['NIFI_NODE']
if 'NIFIREGISTRY' in services:
master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER']
if 'NIFITOOLKITCA' in services:
master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER']
services_configs['NIFITOOLKITCA'] = {
'nifi.toolkit.tls.ca.server.token':
security.get_secret('MASTERKEY')
}
if 'KAFKA' in services:
worker_setups['KAFKA'] = ['KAFKA_BROKER']
if 'SCHEMAREGISTRY' in services:
master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER']
clus_name = clus_name if clus_name else \
horton.cdcred.name + '-' + str(cdh_ver).replace('.', '-')
# Handle virtual instance generation
master_vi = [
create_virtual_instance(tem_name='master',
scripts=[
'''sudo -i
yum install mysql mariadb-server -y
systemctl enable mariadb
service mariadb start
mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8"
mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'"
mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION"
mysql --execute="FLUSH PRIVILEGES"
mysql --execute="COMMIT"'''
])
]
worker_vi = [create_virtual_instance() for _ in range(0, workers)]
try:
cd.ClustersApi(horton.cad).create(
environment=env_name,
deployment=dep_name,
cluster_template=cd.ClusterTemplate(
name=clus_name,
product_versions=products,
parcel_repositories=load_parcels,
services=services,
services_configs=services_configs,
virtual_instance_groups={
'masters':
cd.VirtualInstanceGroup(
name='masters',
min_count=1,
service_type_to_role_types=master_setups,
role_types_configs=master_configs,
virtual_instances=master_vi),
'workers':
cd.VirtualInstanceGroup(
name='workers',
min_count=workers,
service_type_to_role_types=worker_setups,
role_types_configs=worker_configs,
virtual_instances=worker_vi)
}))
except ApiException as e:
if e.status == 409:
log.error("Cluster %s already exists", clus_name)
raise ValueError("Cluster %s already exists", clus_name)
else:
raise e
