def step_2_init_infra(create_wait=0): init_start_ts = _dt.utcnow() log.info("------------- Getting Environment at [%s]", init_start_ts) horton.cbd = infra.get_cloudbreak(purge=horton.global_purge, create_wait=create_wait) log.info("------------- Connecting to Environment") public_ip = horton.cbd.public_ips[0] cbd_url = 'https://' + public_ip + '/cb/api' cad_url = 'https://' + public_ip + ':7189' log.info("Setting Cloudbreak endpoint to %s", cbd_url) utils.set_endpoint(cbd_url) log.info("Setting Altus Director endpoint to %s", cad_url) utils.set_endpoint(cad_url) log.info("------------- Authenticating to Cloudbreak") cbd_auth_success = security.service_login( service='cloudbreak', username=config.profile['email'], password=security.get_secret('ADMINPASSWORD'), bool_response=False) if not cbd_auth_success: raise ConnectionError("Couldn't login to Cloudbreak") else: log.info('Logged into Cloudbreak at [%s]', cbd_url) log.info("------------- Authenticating to Altus Director") cad_auth_success = security.service_login( service='director', username=config.profile['username'], password=security.get_secret('ADMINPASSWORD'), bool_response=False) if not cad_auth_success: raise ConnectionError("Couldn't login to Director") else: log.info('Logged into Director at [%s]', cad_url) # Cloudbreak may have just booted and not be ready for queries yet log.info("Waiting for Cloudbreak API Calls to be available") utils.wait_to_complete(deploy.list_blueprints, bool_response=True, whoville_delay=5, whoville_max_wait=120) # Director may not be ready for queries yet log.info("Waiting for Altus Director API Calls to be available") utils.wait_to_complete(director.list_environments, bool_response=True, whoville_delay=5, whoville_max_wait=120) log.info("------------- Setting Deployment Credential") horton.cred = deploy.get_credential(config.profile['namespace'] + 'credential', create=True, purge=horton.global_purge) init_finish_ts = _dt.utcnow() diff_ts = init_finish_ts - init_start_ts log.info("Completed Infrastructure Init at [%s] after [%d] seconds", init_finish_ts, diff_ts.seconds)
def validate_profile(): log.info("Validating provided profile.yml") horton = Horton() # Check Profile is imported if not config.profile: raise ValueError("whoville Config Profile is not populated with" "deployment controls, cannot proceed") # Check Profile version if 'profilever' not in config.profile: raise ValueError("Your Profile is out of date, please recreate your " "Profile from the template") if config.profile['profilever'] < config.min_profile_ver: raise ValueError("Your Profile is out of date, please recreate your " "Profile from the template. Profile v3 requires an ssh private key or pem file.") # Handle SSH if 'sshkey_file' in config.profile and config.profile['sshkey_file']: assert config.profile['sshkey_file'].endswith('.pem') from Crypto.PublicKey import RSA pem_key = RSA.importKey(fs_read(config.profile['sshkey_file'])) config.profile['sshkey_pub'] = pem_key.publickey().exportKey().decode() config.profile['sshkey_priv'] = pem_key.exportKey().decode() config.profile['sshkey_name'] = os.path.basename(config.profile['sshkey_file']).split('.')[0] else: assert any(k in config.profile for k in ['ssh_key_priv', 'sshkey_priv']) assert all(k in config.profile for k in ['sshkey_pub', 'sshkey_name']) # Check Namespace assert isinstance(horton.namespace, six.string_types),\ "Namespace must be string" assert len(horton.namespace) >= 2,\ "Namespace must be at least 2 characters" # Check Password if 'password' in config.profile and config.profile['password']: horton.cache['ADMINPASSWORD'] = config.profile['password'] else: horton.cache['ADMINPASSWORD'] = security.get_secret('ADMINPASSWORD') password_test = re.compile(r'^(?=.*[A-Za-z])(?=.*\d)[A-Za-z\d-]{12,}$') if not bool(password_test.match(horton.cache['ADMINPASSWORD'])): raise ValueError("Password doesn't match Platform spec." "Requires 12+ characters, at least 1 letter and " "number, may also contain -") # Check Provider provider = config.profile.get('platform')['provider'] assert provider in ['EC2', 'AZURE_ARM', 'GCE'] # TODO: Read in the profile template, check it has all matching keys # Check Profile Namespace is validate ns_test = re.compile(r'[a-z0-9-]') if not bool(ns_test.match(horton.namespace)): raise ValueError("Namespace must only contain 0-9 a-z -") # Check storage bucket matches expected format if 'bucket' in config.profile: if provider == 'EC2': bucket_test = re.compile(r'[a-z0-9.-]') elif provider == 'AZURE_ARM': bucket_test = re.compile(r'[a-z0-9@]') elif provider == 'GCE': bucket_test = re.compile(r'[a-z0-9.-]') else: raise ValueError("Platform Provider not supported") if not bool(bucket_test.match(config.profile['bucket'])): raise ValueError("Bucket name doesn't match Platform spec")
def create_deployment(cm_ver, env_name=None, tem_name=None, dep_name=None, tls_start=False, csds=None): assert isinstance(cm_ver, six.string_types) env_name = env_name if env_name else horton.cdcred.name log.info("Using Environment [%s]", env_name) tem_name = tem_name if tem_name else dep_name if dep_name else horton.cdcred.name log.info("Using Virtual Template [%s]", tem_name) dep_name = dep_name if dep_name else env_name + '-' + str(cm_ver).replace( '.', '-') template = get_instance_template(tem_name=tem_name) if not template: log.info("Template [%s] not found, creating from defaults", tem_name) create_instance_template(tem_name=tem_name) while not template: sleep(2) template = get_instance_template(tem_name=tem_name) if cm_ver[0] == '6': repo = 'https://archive.cloudera.com/cm6/' + cm_ver + '/redhat7/yum/' repo_key = 'https://archive.cloudera.com/cm6/' + cm_ver +\ '/redhat7/yum/RPM-GPG-KEY-cloudera' elif cm_ver[0] == '5': repo = 'https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/' +\ cm_ver + '/' repo_key = 'https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/' \ 'RPM-GPG-KEY-cloudera' else: raise ValueError("Only CM 5 or 6 supported") try: log.info("Attempting to create Deployment [%s]", dep_name) return cd.DeploymentsApi(horton.cad).create( environment=env_name, deployment_template=cd.DeploymentTemplate( name=dep_name, manager_virtual_instance=cd.VirtualInstance(id=str( uuid.uuid4()), template=template), password=security.get_secret('ADMINPASSWORD'), enable_enterprise_trial=True, repository_key_url=repo_key, repository=repo, tls_enabled=tls_start, csds=csds, java_installation_strategy='NONE')) except ApiException as e: if e.status == 409: log.warning('A deployment with the same name already exists') else: raise e
def create_cluster(cluster_def, dep_name, workers=3, env_name=None, scripts=None): env_name = env_name if env_name else horton.cdcred.name cdh_ver = str(cluster_def['products']['CDH']) services = cluster_def['services'] cluster_name = cluster_def['name'] products = cluster_def['products'] if 'post_create_scripts' in cluster_def and cluster_def[ 'post_create_scripts'] is not None: log.info("Including post_create_scripts in cluster of %s", cluster_def['post_create_scripts']) post_create_scripts = [] for script_name in cluster_def['post_create_scripts']: log.info("Adding script %s like [%s]", script_name, scripts[script_name][:50]) post_create_scripts.append(cd.Script(content=scripts[script_name])) else: post_create_scripts = None if cdh_ver[0] == '5': load_parcels = [ 'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/' ] elif cdh_ver[0] == '6': load_parcels = [ 'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/' ] else: raise ValueError("Only CDH versions 5 or 6 supported") if 'parcels' in cluster_def: load_parcels += cluster_def['parcels'] # Default Role and Service configs services_configs = {} master_setups = {} master_configs = {} worker_setups = {} worker_configs = {} if 'HDFS' in services: master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE'] worker_setups['HDFS'] = ['DATANODE'] if 'YARN' in services: master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY'] worker_setups['YARN'] = ['NODEMANAGER'] if 'ZOOKEEPER' in services: master_setups['ZOOKEEPER'] = ['SERVER'] if 'HBASE' in services: master_setups['HBASE'] = ['MASTER'] worker_setups['HBASE'] = ['REGIONSERVER'] if 'HIVE' in services: master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE'] if 'HUE' in services: master_setups['HUE'] = ['HUE_SERVER'] if 'KUDU' in services: master_setups['KUDU'] = ['KUDU_MASTER'] worker_setups['KUDU'] = ['KUDU_TSERVER'] master_configs['KUDU'] = { 'KUDU_MASTER': { 'fs_wal_dir': "/data0/kudu/masterwal", 'fs_data_dirs': "/data1/kudu/master" } } worker_configs['KUDU'] = { 'KUDU_TSERVER': { 'fs_wal_dir': "/data0/kudu/tabletwal", 'fs_data_dirs': "/data1/kudu/tablet" } } if 'IMPALA' in services: master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE'] worker_setups['IMPALA'] = ['IMPALAD'] if 'NIFI' in services: worker_setups['NIFI'] = ['NIFI_NODE'] if 'NIFIREGISTRY' in services: master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER'] if 'NIFITOOLKITCA' in services: master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER'] services_configs['NIFITOOLKITCA'] = { 'nifi.toolkit.tls.ca.server.token': security.get_secret('MASTERKEY') } if 'KAFKA' in services: worker_setups['KAFKA'] = ['KAFKA_BROKER'] services_configs['KAFKA'] = {'producer.metrics.enable': True} if 'SCHEMAREGISTRY' in services: master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER'] # Handle Services Configs overrides if 'servicesconfigs' in cluster_def.keys(): for k, v in cluster_def['servicesconfigs']: services_configs[k] = v # Handle virtual instance generation master_vi = [ create_virtual_instance(tem_name=dep_name + '-' + cluster_name + '-master', scripts=[ '''sudo -i yum install mysql mariadb-server epel-release -y # MariaDB yum -y install npm gcc-c++ make # SMM-UI npm install forever -g # SMM-UI systemctl enable mariadb service mariadb start mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8" mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'" mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'" mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION" mysql --execute="CREATE DATABASE streamsmsgmgr DEFAULT CHARACTER SET utf8" mysql --execute="CREATE USER 'streamsmsgmgr'@'localhost' IDENTIFIED BY 'streamsmsgmgr'" mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' identified by 'streamsmsgmgr'" mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'%' WITH GRANT OPTION" mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' identified by 'streamsmsgmgr'" mysql --execute="GRANT ALL PRIVILEGES ON streamsmsgmgr.* TO 'streamsmsgmgr'@'localhost' WITH GRANT OPTION" mysql --execute="FLUSH PRIVILEGES" mysql --execute="COMMIT"''' ]) ] worker_vi = [ create_virtual_instance(tem_name=dep_name + '-' + cluster_name + '-worker') for _ in range(0, workers) ] try: cd.ClustersApi(horton.cad).create( environment=env_name, deployment=dep_name, cluster_template=cd.ClusterTemplate( name=cluster_name, product_versions=products, parcel_repositories=load_parcels, services=services, services_configs=services_configs, virtual_instance_groups={ 'masters': cd.VirtualInstanceGroup( name='masters', min_count=1, service_type_to_role_types=master_setups, role_types_configs=master_configs, virtual_instances=master_vi), 'workers': cd.VirtualInstanceGroup( name='workers', min_count=workers, service_type_to_role_types=worker_setups, role_types_configs=worker_configs, virtual_instances=worker_vi) }, post_create_scripts=post_create_scripts)) except ApiException as e: if e.status == 409: log.error("Cluster %s already exists", cluster_name) raise ValueError("Cluster %s already exists", cluster_name) else: raise e
def init_cbreak_infra(create=True, create_wait=0): init_start_ts = _dt.utcnow() log.info("------------- Getting Environment at [%s]", init_start_ts) horton.cbd = infra.get_cloudbreak(purge=horton.global_purge, create_wait=create_wait, create=create) if not horton.cbd: if create: # Create has failed, throw error raise ValueError( "Cloudbreak Create requested but failed, exiting...") else: return None else: log.info("Found existing Cloudbreak in Namespace, connecting...") log.info("------------- Connecting to Environment") if horton.cbd.public_ips: public_ip = horton.cbd.public_ips[0] else: public_ip = horton.cbd.name + config.profile['platform']['domain'] cbd_url = 'https://' + public_ip + '/cb/api' cad_url = 'https://' + public_ip + ':7189' log.info("Setting Cloudbreak endpoint to %s", cbd_url) utils.set_endpoint(cbd_url) log.info("Setting Altus Director endpoint to %s", cad_url) utils.set_endpoint(cad_url) log.info("------------- Authenticating to Cloudbreak") cbd_auth_success = security.service_login( service='cloudbreak', username=config.profile['email'], password=security.get_secret('ADMINPASSWORD'), bool_response=False) if not cbd_auth_success: raise ConnectionError("Couldn't login to Cloudbreak") else: log.info('Logged into Cloudbreak at [%s]', cbd_url) log.info("------------- Authenticating to Altus Director") cad_auth_success = security.service_login( service='director', username=config.profile['username'], password=security.get_secret('ADMINPASSWORD'), bool_response=False) if not cad_auth_success: raise ConnectionError("Couldn't login to Director") else: log.info('Logged into Director at [%s]', cad_url) # Cloudbreak may have just booted and not be ready for queries yet log.info("Waiting for Cloudbreak API Calls to be available") utils.wait_to_complete(deploy.list_blueprints, bool_response=True, whoville_delay=5, whoville_max_wait=120) # Director may not be ready for queries yet log.info("Waiting for Altus Director API Calls to be available") utils.wait_to_complete(director.list_environments, bool_response=True, whoville_delay=5, whoville_max_wait=120) # Validating Cloudbreak version if not deploy.check_cloudbreak_version(): raise ValueError( "Cloudbreak server is older than configured minimum version of %s", str(config.cb_ver)) # Creating Environment Credentials log.info("------------- Setting Deployment Credential") log.info("Ensuring Credential for Cloudbreak") horton.cbcred = deploy.get_credential(config.profile['namespace'] + 'credential', create=True, purge=horton.global_purge) if ((config.profile['platform']['provider'] == 'EC2') or (config.profile['platform']['provider'] == 'GCE')): log.info("Ensuring Environment Credential for Director") horton.cadcred = director.get_environment() init_finish_ts = _dt.utcnow() diff_ts = init_finish_ts - init_start_ts log.info("Completed Infrastructure Init at [%s] after [%d] seconds", init_finish_ts, diff_ts.seconds)
def create_cloudbreak(session, cbd_name): public_ip = requests.get('https://ipv4.icanhazip.com').text.rstrip() net_rules = [{ 'protocol': 'tcp', 'from_port': 9443, 'to_port': 9443, 'cidr_ips': ['0.0.0.0/0'] }, { 'protocol': -1, 'from_port': 1, 'to_port': 65535, 'cidr_ips': [public_ip + '/32'] }, { 'protocol': 'tcp', 'from_port': 443, 'to_port': 443, 'cidr_ips': ['0.0.0.0/0'] }, { 'protocol': 'tcp', 'from_port': 22, 'to_port': 22, 'cidr_ips': ['0.0.0.0/0'] }] if session.type == 'ec2': s_boto3 = create_boto3_session() aws_clean_cloudformation(s_boto3) log.info("Selecting OS Image for Cloudbreak") images = list_images(session, filters={ 'name': '*CentOS Linux 7 x86_64 HVM EBS ENA*', }) image = sorted(images, key=lambda k: k.extra['description'][-7:]) if not image: raise ValueError("Couldn't find a valid Centos7 Image") else: image = image[-1] bd = image.extra['block_device_mapping'][0] root_vol = { 'VirtualName': None, 'DeviceName': bd['device_name'], 'Ebs': { 'VolumeSize': 50, 'VolumeType': bd['ebs']['volume_type'], 'DeleteOnTermination': True } } log.info("Fetching list of suitable machine types") machines = list_sizes_aws(session, cpu_min=4, cpu_max=4, mem_min=16000, mem_max=20000) if not machines: raise ValueError("Couldn't find a VM of the right size") else: machine = machines[-1] log.info("Fetching list of available networks") networks = list_networks(session) network = sorted(networks, key=lambda k: k.extra['is_default']) if not network: raise ValueError("There should be at least one network, this " "is rather unexpected") else: network = network[-1] log.info("Fetching subnets in Network") subnets = list_subnets(session, {'extra.vpc_id': network.id}) subnets = sorted(subnets, key=lambda k: k.state) ec2_resource = s_boto3.resource('ec2') if not subnets: raise ValueError("Expecting at least one subnet on a network") subnet = [ x for x in subnets if ec2_resource.Subnet(x.id).map_public_ip_on_launch ] if not subnet: raise ValueError("There are no subnets with auto provisioning of " "public IPs enabled..." "enable public IP auto provisioning on at least " "one subnet in the default VPC") else: subnet = subnet[0] log.info("Fetching Security groups matching namespace") sec_group = list_security_groups(session, {'name': namespace}) if not sec_group: log.info("Namespace Security group not found, creating") _ = session.ex_create_security_group( name=namespace + 'whoville-default', description=namespace + 'whoville-default Security Group', vpc_id=network.id) sec_group = list_security_groups(session, {'name': namespace})[-1] else: sec_group = sec_group[-1] net_rules.append({ 'protocol': -1, 'group_pairs': [{ 'group_id': sec_group.id }], 'from_port': 0, 'to_port': 0 }) # security group loopback doesn't work well on AWS, need to use subnet net_rules.append({ 'protocol': -1, 'cidr_ips': [subnet.extra['cidr_block']], 'from_port': 0, 'to_port': 0 }) for rule in net_rules: add_sec_rule_to_ec2_group(session, rule, sec_group.id) log.info("Checking for expected SSH Keypair") ssh_key = list_keypairs(session, {'name': config.profile['sshkey_name']}) if not ssh_key: ssh_key = session.import_key_pair_from_string( name=config.profile['sshkey_name'], key_material=config.profile['sshkey_pub']) else: ssh_key = [ x for x in ssh_key if x.name == config.profile['sshkey_name'] ][0] log.info("Creating Static IP for Cloudbreak") try: static_ips = [ x for x in session.ex_describe_all_addresses() if x.instance_id is None ] except InvalidCredsError: static_ips = None if not static_ips: static_ip = session.ex_allocate_address() else: static_ip = static_ips[0] if not static_ip: raise ValueError("Couldn't get a Static IP for Cloudbreak") # This is just a tidy way of specifying a script cb_ver = config.profile.get('cloudbreak_ver') cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver script_lines = [ "#!/bin/bash", "cd /root", "export cb_ver=" + cb_ver, "export uaa_secret=" + security.get_secret('MASTERKEY'), "export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'), "export uaa_default_email=" + config.profile['email'], "export public_ip=" + static_ip.ip, "source <(curl -sSL https://raw.githubusercontent.com/Chaffelson" "/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)" ] script = '\n'.join(script_lines) cbd = create_node(session=session, name=cbd_name, image=image, machine=machine, params={ 'ex_security_group_ids': [sec_group.id], 'ex_subnet': subnet, 'ex_assign_public_ip': True, 'ex_blockdevicemappings': [root_vol], 'ex_keyname': ssh_key.name, 'ex_userdata': script }) # inserting hard wait to bypass race condition where returned node ID # is not actually available to the list API call yet sleep(5) log.info("Waiting for Cloudbreak Instance to be Available...") session.wait_until_running(nodes=[cbd]) log.info("Cloudbreak Infra Booted at [%s]", cbd) log.info("Assigning Static IP to Cloudbreak") session.ex_associate_address_with_node(cbd, static_ip) # Assign Role ARN if 'infraarn' in config.profile['platform']: log.info("Found infraarn in Profile, associating with Cloudbreak") infra_arn = config.profile['platform']['infraarn'] client = s_boto3.client('ec2') client.associate_iam_instance_profile(IamInstanceProfile={ 'Arn': infra_arn, 'Name': infra_arn.rsplit('/')[-1] }, InstanceId=cbd.id) # get updated node information cbd = list_nodes(session, {'name': cbd_name}) cbd = [x for x in cbd if x.state == 'running'] if cbd: return cbd[0] else: raise ValueError("Failed to create new Cloubreak Instance") if session.type == 'azure_arm': ssh_key = config.profile['sshkey_pub'] resource_group = namespace + 'cloudbreak-group' network_name = namespace + 'cloudbreak-network' subnet_name = namespace + 'cloudbreak-subnet' sec_group_name = namespace + 'cloudbreak-secgroup' public_ip_name = namespace + 'cloudbreak-ip' nic_name = namespace + 'cloudbreak-nic' disk_account_name = namespace + 'diskaccount' disk_account_name = disk_account_name.replace('-', '') # ToDo: examine cleaning Azure Resource Groups log.info("Creating Resource Group...") token = get_azure_token() azure_resource_client = create_azure_session(token, 'resource') azure_resource_client.resource_groups.create_or_update( resource_group, {'location': config.profile.get('platform')['region']}) image = session.list_images(ex_publisher='OpenLogic', ex_offer='CentOS-CI', ex_sku='7-CI') if not image: raise ValueError("Couldn't find a valid Centos7 Image") else: image = image[-1] machines = list_sizes_azure(session, cpu_min=4, cpu_max=4, mem_min=16384, mem_max=20480) if not machines: raise ValueError("Couldn't find a VM of the right size") else: machine = machines[0] log.info("Checking for disk storage account, please wait...") azure_storage_client = create_azure_session(token, 'storage') try: azure_storage_client.storage_accounts.create( resource_group, disk_account_name, { 'location': config.profile.get('platform')['region'], 'sku': { 'name': 'standard_lrs' }, 'kind': 'StorageV2' }).wait() except Exception: log.info("Found existing os disk account...") log.info("Looking for existing network resources...") azure_network_client = create_azure_session(token, 'network') try: log.info("Getting Vnet...") network = azure_network_client.virtual_networks.get( resource_group, network_name) log.info("Getting Network Interface...") nic = azure_network_client.network_interfaces.get( resource_group, nic_name) log.info("Getting Public IP...") public_ip = azure_network_client.public_ip_addresses.get( resource_group, public_ip_name) except Exception: log.info("No Vnet exists for this namepsace, creating...") network = azure_network_client.virtual_networks.create_or_update( resource_group, network_name, { 'location': config.profile.get('platform')['region'], 'address_space': { 'address_prefixes': ['10.0.0.0/16'] } }) network = network.result() log.info("Creating Subnet...") subnet = azure_network_client.subnets.create_or_update( resource_group, network_name, subnet_name, {'address_prefix': '10.0.0.0/24'}) log.info("Creating Public IP...") public_ip = azure_network_client.public_ip_addresses.create_or_update( resource_group, public_ip_name, { 'location': config.profile.get('platform')['region'], 'public_ip_allocation_method': 'static' }) subnet = subnet.result() public_ip = public_ip.result() log.info("Creating Security Group...") sec_group = azure_network_client.network_security_groups.create_or_update( resource_group, sec_group_name, { 'location': config.profile.get('platform')['region'], 'security_rules': [{ 'name': 'ssh_rule', 'description': 'Allow SSH', 'protocol': 'Tcp', 'source_port_range': '*', 'destination_port_range': '22', 'source_address_prefix': 'Internet', 'destination_address_prefix': '*', 'access': 'Allow', 'priority': 100, 'direction': 'Inbound' }, { 'name': 'http_rule', 'description': 'Allow HTTP', 'protocol': 'Tcp', 'sourcePortRange': '*', 'destinationPortRange': '80', 'sourceAddressPrefix': 'Internet', 'destinationAddressPrefix': '*', 'access': 'Allow', 'priority': 101, 'direction': 'Inbound' }, { 'name': 'https_rule', 'provisioningState': 'Succeeded', 'description': 'Allow HTTPS', 'protocol': 'Tcp', 'sourcePortRange': '*', 'destinationPortRange': '443', 'sourceAddressPrefix': 'Internet', 'destinationAddressPrefix': '*', 'access': 'Allow', 'priority': 102, 'direction': 'Inbound' }, { 'name': 'knox_https_rule', 'provisioningState': 'Succeeded', 'description': 'Allow CB HTTPS', 'protocol': 'Tcp', 'sourcePortRange': '*', 'destinationPortRange': '8443', 'sourceAddressPrefix': 'Internet', 'destinationAddressPrefix': '*', 'access': 'Allow', 'priority': 103, 'direction': 'Inbound' }, { 'name': 'cb_https_rule', 'provisioningState': 'Succeeded', 'description': 'Allow CB HTTPS', 'protocol': 'Tcp', 'sourcePortRange': '*', 'destinationPortRange': '9443', 'sourceAddressPrefix': 'Internet', 'destinationAddressPrefix': '*', 'access': 'Allow', 'priority': 104, 'direction': 'Inbound' }, { 'name': 'altus_http_rule', 'provisioningState': 'Succeeded', 'description': 'Allow Altus HTTP', 'protocol': 'Tcp', 'sourcePortRange': '*', 'destinationPortRange': '7189', 'sourceAddressPrefix': 'Internet', 'destinationAddressPrefix': '*', 'access': 'Allow', 'priority': 105, 'direction': 'Inbound' }] }) sec_group = sec_group.result() log.info("Creating Network Interface...") nic = azure_network_client.network_interfaces.create_or_update( resource_group, nic_name, { 'location': config.profile.get('platform')['region'], 'network_security_group': { 'id': sec_group.id }, 'ip_configurations': [{ 'name': 'default', 'subnet': { 'id': subnet.id }, 'public_ip_address': { 'id': public_ip.id } }] }) nic = nic.result() # End network exception handling for missing vnet public_ip = public_ip.ip_address cb_ver = config.profile.get('cloudbreak_ver') cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver script_lines = [ "#!/bin/bash", "cd /root", "yum install -y wget", "wget -O jq https://github.com/stedolan/jq/releases/download/" "jq-1.5/jq-linux64", "chmod +x ./jq", "cp jq /usr/bin", "export cb_ver=" + cb_ver, "export uaa_secret=" + security.get_secret('MASTERKEY'), "export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'), "export uaa_default_email=" + config.profile['email'], "export public_ip=" + public_ip, "source <(curl -sSL https://raw.githubusercontent.com/Chaffelson" "/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)" ] script = '\n'.join(script_lines) script = script.encode() script = str(base64.urlsafe_b64encode(script))\ .replace("b'","").replace("'","") log.info("Creating Virtual Machine...") log.info("with custom_data string like: " + script[:100]) azure_compute_client = create_azure_session(token, 'compute') cbd = azure_compute_client.virtual_machines.create_or_update( resource_group, cbd_name, { 'location': config.profile.get('platform')['region'], 'os_profile': { 'computer_name': cbd_name, 'admin_username': '******', 'linux_configuration': { 'disable_password_authentication': True, 'ssh': { 'public_keys': [{ 'path': '/home/{}/.ssh/authorized_keys'.format( 'centos'), 'key_data': ssh_key }] } }, 'custom_data': script }, 'hardware_profile': { 'vm_size': 'Standard_DS3_v2' }, 'storage_profile': { 'image_reference': { 'publisher': 'Redhat', 'offer': 'RHEL', 'sku': '7-RAW-CI', 'version': 'latest' }, 'os_disk': { 'name': cbd_name, 'create_option': 'fromImage', 'vhd': { 'uri': 'https://{}.blob.core.windows.net/' 'vhds/{}.vhd'.format(disk_account_name, cbd_name) } }, }, 'network_profile': { 'network_interfaces': [{ 'id': nic.id, 'primary': True }] } }) log.info("Waiting for Cloudbreak Instance to be Available...") cbd.wait() cbd = list_nodes(session, {'name': cbd_name}) cbd = [x for x in cbd if x.state == 'running'] if cbd: return cbd[0] else: raise ValueError("Failed to create new Cloubreak Instance") elif session.type == 'gce': region = config.profile['platform']['region'] cbd_name = namespace + 'cloudbreak' public_ip_name = namespace + 'cloudbreak-public-ip' subnet_name = namespace + 'cloudbreak-subnet' firewall_name = namespace + 'cloudbreak-secgroup' ssh_key = config.profile['sshkey_pub'] log.info("Looking for existing network...") networks = session.ex_list_networks() network = [x for x in networks if x.mode == 'auto'] if not network: raise ValueError("There should be at least one network") else: network = network[-1] log.info("Found network: " + network.name) log.info("Looking for existing subnets...") subnets = session.ex_list_subnetworks(region=region) subnet = [x for x in subnets if x.name == 'default'] if not subnet: session.ex_create_subnetwork(name=subnet_name, region=region, network=network) else: subnet = subnet[-1] subnet_name = subnet.name log.info("Found existing subnet called: " + subnet_name) log.info("Getting Public IP...") try: public_ip = session.ex_get_address(name=public_ip_name, region=region) log.info("Found existing Public IP matching name: " + public_ip_name) except ResourceNotFoundError: public_ip = session.ex_create_address(name=public_ip_name, region=region) log.info("Creating new Public IP with name: " + public_ip_name) images = session.list_images() image = [ x for x in images if x.extra['family'] == 'centos-7' and 'centos-7' in x.name ] zones = session.ex_list_zones() zone = [x for x in zones if region in x.name and x.status == 'UP'] if not zone: raise ValueError("Couldn't find a zone for the requested region..") else: zone = zone[-1] if not image: raise ValueError("Couldn't find a valid Centos7 Image") else: image = image[-1] machines = list_sizes_gce(session, location=region, cpu_min=4, cpu_max=4, mem_min=13000, mem_max=20000) if not machines: raise ValueError("Couldn't find a VM of the right size") else: machine = machines[-1] log.info("Creating Security Group...") try: _ = session.ex_get_firewall(name=firewall_name) log.info("Found existing firewall definition called: " + firewall_name) except ResourceNotFoundError: log.info("Creating new firewall definition called: " + firewall_name) net_rules = [{ 'IPProtocol': 'tcp', 'ports': ['22', '443', '8443', '9443', '7189'] }] _ = session.ex_create_firewall(name=firewall_name, network=network, allowed=net_rules, target_tags=[cbd_name]) cb_ver = config.profile.get('cloudbreak_ver') cb_ver = str(cb_ver) if cb_ver else preferred_cb_ver script_lines = [ "#!/bin/bash", "cd /root", "export cb_ver=" + cb_ver, "export uaa_secret=" + security.get_secret('MASTERKEY'), "export uaa_default_pw=" + security.get_secret('ADMINPASSWORD'), "export uaa_default_email=" + config.profile['email'], "export public_ip=" + public_ip.address, "source <(curl -sSL https://raw.githubusercontent.com/Chaffelson" "/whoville/master/bootstrap/v2/cbd_bootstrap_centos7.sh)" ] script = '\n'.join(script_lines) metadata = { 'items': [{ 'key': 'startup-script', 'value': script }, { 'key': 'ssh-keys', 'value': 'centos:' + ssh_key }] } log.info("Creating Cloudbreak instance...") cbd = session.create_node(name=cbd_name, size=machine, image=image, location=zone, ex_network=network, external_ip=public_ip, ex_metadata=metadata, ex_tags=[cbd_name]) log.info("Waiting for Cloudbreak Instance to be Available...") session.wait_until_running(nodes=[cbd]) cbd = list_nodes(session, {'name': cbd_name}) cbd = [x for x in cbd if x.state == 'running'] if cbd: return cbd[0] else: raise ValueError("Failed to create new Cloubreak Instance") else: raise ValueError("Cloudbreak AutoDeploy only supported on EC2, Azure, " "and GCE")
def create_cluster(cdh_ver, workers=3, services=None, env_name=None, dep_name=None, clus_name=None, parcels=None): env_name = env_name if env_name else horton.cdcred.name dep_name = dep_name if dep_name else env_name + '-' + cdh_ver.replace( '.', '-') services = services if services else ['HDFS', 'YARN'] if cdh_ver[0] == '5': load_parcels = [ 'https://archive.cloudera.com/cdh5/parcels/' + cdh_ver + '/' ] elif cdh_ver[0] == '6': load_parcels = [ 'https://archive.cloudera.com/cdh6/' + cdh_ver + '/parcels/' ] else: raise ValueError("Only CDH versions 5 or 6 supported") if parcels: load_parcels += parcels if cdh_ver[0] == '6': # CDH6 already has Kafka load_parcels = [x for x in load_parcels if 'kafka' not in x] products = {'CDH': cdh_ver} if 'NIFI' in str(services): products['CFM'] = '1' if 'KAFKA' in str(services) and cdh_ver[0] == '5': products['KAFKA'] = '4' if 'SCHEMAREGISTRY' in str(services): products['SCHEMAREGISTRY'] = '0.7' services_configs = {} master_setups = {} master_configs = {} worker_setups = {} worker_configs = {} if 'HDFS' in services: master_setups['HDFS'] = ['NAMENODE', 'SECONDARYNAMENODE'] worker_setups['HDFS'] = ['DATANODE'] if 'YARN' in services: master_setups['YARN'] = ['RESOURCEMANAGER', 'JOBHISTORY'] worker_setups['YARN'] = ['NODEMANAGER'] if 'ZOOKEEPER' in services: master_setups['ZOOKEEPER'] = ['SERVER'] if 'HBASE' in services: master_setups['HBASE'] = ['MASTER'] worker_setups['HBASE'] = ['REGIONSERVER'] if 'HIVE' in services: master_setups['HIVE'] = ['HIVESERVER2', 'HIVEMETASTORE'] if 'HUE' in services: master_setups['HUE'] = ['HUE_SERVER'] if 'KUDU' in services: master_setups['KUDU'] = ['KUDU_MASTER'] worker_setups['KUDU'] = ['KUDU_TSERVER'] master_configs['KUDU'] = { 'KUDU_MASTER': { 'fs_wal_dir': "/data0/kudu/masterwal", 'fs_data_dirs': "/data1/kudu/master" } } worker_configs['KUDU'] = { 'KUDU_TSERVER': { 'fs_wal_dir': "/data0/kudu/tabletwal", 'fs_data_dirs': "/data1/kudu/tablet" } } if 'IMPALA' in services: master_setups['IMPALA'] = ['CATALOGSERVER', 'STATESTORE'] worker_setups['IMPALA'] = ['IMPALAD'] if 'NIFI' in services: worker_setups['NIFI'] = ['NIFI_NODE'] if 'NIFIREGISTRY' in services: master_setups['NIFIREGISTRY'] = ['NIFI_REGISTRY_SERVER'] if 'NIFITOOLKITCA' in services: master_setups['NIFITOOLKITCA'] = ['NIFI_TOOLKIT_SERVER'] services_configs['NIFITOOLKITCA'] = { 'nifi.toolkit.tls.ca.server.token': security.get_secret('MASTERKEY') } if 'KAFKA' in services: worker_setups['KAFKA'] = ['KAFKA_BROKER'] if 'SCHEMAREGISTRY' in services: master_setups['SCHEMAREGISTRY'] = ['SCHEMA_REGISTRY_SERVER'] clus_name = clus_name if clus_name else \ horton.cdcred.name + '-' + str(cdh_ver).replace('.', '-') # Handle virtual instance generation master_vi = [ create_virtual_instance(tem_name='master', scripts=[ '''sudo -i yum install mysql mariadb-server -y systemctl enable mariadb service mariadb start mysql --execute="CREATE DATABASE registry DEFAULT CHARACTER SET utf8" mysql --execute="CREATE USER 'registry'@'localhost' IDENTIFIED BY 'registry'" mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' identified by 'registry'" mysql --execute="GRANT ALL PRIVILEGES ON registry.* TO 'registry'@'localhost' WITH GRANT OPTION" mysql --execute="FLUSH PRIVILEGES" mysql --execute="COMMIT"''' ]) ] worker_vi = [create_virtual_instance() for _ in range(0, workers)] try: cd.ClustersApi(horton.cad).create( environment=env_name, deployment=dep_name, cluster_template=cd.ClusterTemplate( name=clus_name, product_versions=products, parcel_repositories=load_parcels, services=services, services_configs=services_configs, virtual_instance_groups={ 'masters': cd.VirtualInstanceGroup( name='masters', min_count=1, service_type_to_role_types=master_setups, role_types_configs=master_configs, virtual_instances=master_vi), 'workers': cd.VirtualInstanceGroup( name='workers', min_count=workers, service_type_to_role_types=worker_setups, role_types_configs=worker_configs, virtual_instances=worker_vi) })) except ApiException as e: if e.status == 409: log.error("Cluster %s already exists", clus_name) raise ValueError("Cluster %s already exists", clus_name) else: raise e