def read_aws_region(self): settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': self.aws_region = input("\n\tPlease Enter AWS Region: ") Settings.set('AWS_REGION', self.aws_region) else: self.aws_region = settings_region
def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "roles")) self.need_complete_install = self._need_complete_installation() Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) super().__init__(args)
class ApplicationLoadBalancer(LoadBalancerResource): name = "" internal = Settings.get('MAKE_ALB_INTERNAL', True) load_balancer_type = "application" security_groups = [InfraSecurityGroupResource.get_output_attr('id')] subnets = Settings.get('VPC')['SUBNETS'] OUTPUT_LIST = ['dns_name'] @classmethod def get_http_url(cls): return "http://%s" % cls.get_output_attr('dns_name') @classmethod def get_api_base_url(cls): return "http://%s/api" % cls.get_output_attr('dns_name') @classmethod def get_api_version_url(cls, service): version_url = cls.get_api_server_url(service) return version_url if service == "auth" else version_url + "/v1" @classmethod def get_api_server_url(cls, service): return "%s/%s" % (cls.get_api_base_url(), service) def render_output(self, outputs): if self.resource_in_tf_output(outputs): return { 'Pacbot Domain': outputs[self.get_resource_id()]['dns_name'], 'Admin': Settings.PACBOT_LOGIN_CREDENTIALS['Admin'], 'User': Settings.PACBOT_LOGIN_CREDENTIALS['User'] }
def validate_subnet_ids(self): self.error_message = None if Settings.get('VPC', None): vpc_ids = [Settings.VPC['ID']] if Settings.VPC.get('SUBNETS', None): if Settings.get('REQUIRE_SUBNETS_ON_DIFFERENT_ZONE', False): subnet_ids = Settings.VPC['SUBNETS'] try: valid_subnets = vpc.get_vpc_subnets( Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY, Settings.AWS_REGION, vpc_ids) except Exception as e: self.error_message = str( e) + "\n\t" + K.CORRECT_VPC_MSG return K.NOT_VALID current_subnets = [ subnet for subnet in valid_subnets if subnet['SubnetId'] in subnet_ids ] if len(current_subnets) != len(subnet_ids): self.error_message = K.INVALID_SUBNETS return K.NOT_VALID if len( set([ subnet['AvailabilityZone'] for subnet in current_subnets ])) < 2: self.error_message = K.INVALID_SUBNET_ZONES return K.NOT_VALID return K.VALID
def __init__(self, args): Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) self.destroy_resource_tags_list = [ v for (k, v) in args if k == self.category_field_name ] args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "roles")) args.append((K.CATEGORY_FIELD_NAME, "all_read_role")) args.append((K.CATEGORY_FIELD_NAME, "batch-ecr")) args.append((K.CATEGORY_FIELD_NAME, "batch-job")) args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) self.reinstall_resource_tags_list = [ v for (k, v) in args if k == self.category_field_name ] self.need_complete_install = self._need_complete_installation() self.dry_run = True if any([x[1] for x in args if x[0] == "dry-run"]) else self.dry_run self.silent_install = True if any( [x[1] for x in args if x[0] == "silent"]) else self.silent_install
def load_aws_account_details(self): """Find AWS Account ID from the credentials given""" caller_identity = get_aws_caller_identity(self.AWS_AUTH_CRED) Settings.set('AWS_ACCOUNT_ID', caller_identity.get('Account')) Settings.set('CALLER_ARN', caller_identity.get('Arn')) self.AWS_ACCOUNT_ID = caller_identity.get('Account') self.CALLER_ARN = caller_identity.get('Arn')
def load_aws_account_id(self): """Find AWS Account ID from the credentials given""" aws_account_id = get_user_account_id(Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY, Settings.AWS_SESSION_TOKEN) Settings.set('AWS_ACCOUNT_ID', aws_account_id) self.aws_account_id = aws_account_id
def get_pacbot_domain_url(cls): pacbot_domain = Settings.get('PACBOT_DOMAIN', None) pacbot_domain = pacbot_domain if pacbot_domain else cls.get_output_attr( 'dns_name') return "%s://%s" % (Settings.get('ALB_PROTOCOL', "HTTP").lower(), pacbot_domain)
def read_aws_region(self): settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': self.aws_region = input("\n\t%s" % K.AWS_REGION_INPUT) Settings.set('AWS_REGION', self.aws_region) else: self.aws_region = settings_region
def validate(self): self.show_step_heading(K.SETTINGS_CHECK_STARTED) status = self.validate_vpc_and_cidr_blocks() self.show_step_inner_messaage(K.VPC_CHECK_STARTED, status, self.error_message) if status != K.VALID: return False status = self.validate_subnet_ids() self.show_step_inner_messaage(K.SUBNETS_CHECK_STARTED, status, self.error_message) if status != K.VALID: return False status = self.validate_policies() if Settings.get('MINIMUM_RAM', None): status = self.validate_system_ram_config() self.show_step_inner_messaage(K.SYSTEM_RAM_CONFIG_CHECK_STARTED, status, self.error_message) if status != K.VALID: return False if Settings.get('MINIMUM_STORAGE', None): status = self.validate_system_storage_config() self.show_step_inner_messaage(K.SYSTEM_STORAGE_CONFIG_CHECK_STARTED, status, self.error_message) if status != K.VALID: return False return status
class InfraSecurityGroupResource(SecurityGroupResource): name = "" vpc_id = Settings.get('VPC')['ID'] ingress = [{ 'from_port': 0, 'to_port': 0, 'protocol': "-1", 'cidr_blocks': Settings.get('VPC')['CIDR_BLOCKS'], 'ipv6_cidr_blocks': [], 'prefix_list_ids': [], 'description': "", 'self': False, 'security_groups': [] }] egress = [{ 'from_port': 0, 'to_port': 0, 'protocol': "-1", 'cidr_blocks': ["0.0.0.0/0"], 'ipv6_cidr_blocks': [], 'prefix_list_ids': [], 'description': "", 'self': False, 'security_groups': [] }]
def validate_subnet_ids(self): """ Check the subnets provided are present under the given VPC or not Returns: valid or not_valid (str): Configured string for valid and not valid conditions """ self.error_message = None if Settings.get('VPC', None): vpc_ids = [Settings.VPC['ID']] if Settings.VPC.get('SUBNETS', None): if Settings.get('REQUIRE_SUBNETS_ON_DIFFERENT_ZONE', False): subnet_ids = Settings.VPC['SUBNETS'] try: valid_subnets = vpc.get_vpc_subnets( vpc_ids, Settings.AWS_AUTH_CRED ) except Exception as e: self.error_message = str(e) + "\n\t" + K.CORRECT_VPC_MSG return K.NOT_VALID current_subnets = [subnet for subnet in valid_subnets if subnet['SubnetId'] in subnet_ids] if len(current_subnets) != len(subnet_ids): self.error_message = K.INVALID_SUBNETS return K.NOT_VALID if len(set([subnet['AvailabilityZone'] for subnet in current_subnets])) < 2: self.error_message = K.INVALID_SUBNET_ZONES return K.NOT_VALID return K.VALID
def read_aws_region(self): """Read AWS region from user if it is not already set in settings""" settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': self.aws_region = input("\n\t%s" % K.AWS_REGION_INPUT) Settings.set('AWS_REGION', self.aws_region) else: self.aws_region = settings_region
def load_settings(self, config_path): """ Load all the main and local configurations into the system Args: config_path (str): This is the path to the main configuration/settings file """ Settings.load_setings(config_path)
def show_step_heading(self, heading, write_log=True): if write_log: SysLog().write_debug_log(heading) step_count_num = Settings.get('step_count_num', 1) print( self._get_heading_message_in_color( "\nStep %s: %s" % (str(step_count_num), heading), self.BCYAN_ANSI)) step_count_num = Settings.set('step_count_num', step_count_num + 1)
class RuleEngineBatchJobEnv(BatchComputeEnvironmentResource): compute_environment_name = "" instance_role = ECSRoleInstanceProfile.get_output_attr('arn') instance_type = [Settings.get('BATCH_INSTANCE_TYPE', "m4.xlarge")] max_vcpus = 256 min_vcpus = 0 desired_vcpus = 0 ec2_key_pair = "" resource_type = "EC2" security_group_ids = [InfraSecurityGroupResource.get_output_attr('id')] subnets = Settings.get('VPC')['SUBNETS'] env_type = "MANAGED" service_role = BatchRole.get_output_attr('arn') compute_resources_tags = get_all_resource_tags() DEPENDS_ON = [ BatchIAMRolePolicyAttach ] # This is required otherwise policy would be dettached from Batchrole def pre_terraform_apply(self): ec2_client = get_ec2_client(self.input.AWS_AUTH_CRED) ec2_key_pair = self.get_input_attr('ec2_key_pair') try: key_obj = ec2_client.create_key_pair(KeyName=ec2_key_pair) with open(os.path.join(Settings.OUTPUT_DIR, ec2_key_pair + ".pem"), "w") as keyfile: keyfile.write(key_obj['KeyMaterial']) except Exception as e: pass def check_batch_jobs_running(self): envs = get_compute_environments( [self.get_input_attr('compute_environment_name')], self.input.AWS_AUTH_CRED) if not len(envs): return if envs[0]['computeResources']['desiredvCpus'] > int( self.get_input_attr('desired_vcpus')): return True def pre_generate_terraform(self): warn_msg = "Batch Jobs are running, please try after it gets completed and desired CPUs comes to 0." if self.check_batch_jobs_running(): message = "\n\t ** %s **\n" % warn_msg print(MsgMixin.BERROR_ANSI + message + MsgMixin.RESET_ANSI) sys.exit() def post_terraform_destroy(self): ec2_client = get_ec2_client(self.input.AWS_AUTH_CRED) ec2_key_pair = self.get_input_attr('ec2_key_pair') try: key_obj = ec2_client.delete_key_pair(KeyName=ec2_key_pair) except Exception as e: print(ec2_key_pair + " Not able to delete Key Pair. Error: %s" % str(e))
def read_aws_access_key(self): settings_access_key = getattr(Settings, 'AWS_ACCESS_KEY', None) if settings_access_key is None or settings_access_key == '': self.aws_access_key = input("\n\t%s" % K.AWS_ACCESS_KEY_INPUT) if len(self.aws_access_key) < 20: self.show_step_inner_error("\n\t" + K.INVALID_KEY) raise Exception(K.INVALID_KEY) Settings.set('AWS_ACCESS_KEY', self.aws_access_key) else: self.aws_access_key = settings_access_key
def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "batch-ecr")) args.append((K.CATEGORY_FIELD_NAME, "batch-job")) args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) super().__init__(args)
def __init__(self, args): # args.append((K.CATEGORY_FIELD_NAME, "datastore")) # tf_outputs = PyTerraform.load_terraform_output_from_json_file() # role_file = os.path.join(Settings.TERRAFORM_DIR, "iam_all_read_role_AllReadRole.tf") # if not tf_outputs.get(AllReadRole.get_resource_id(), False): # args.append((K.CATEGORY_FIELD_NAME, "all_read_role")) # args.append((K.CATEGORY_FIELD_NAME, "ecs_role")) Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) super().__init__(args)
def read_aws_access_key(self): settings_access_key = getattr(Settings, 'AWS_ACCESS_KEY', None) if settings_access_key is None or settings_access_key == '': self.aws_access_key = input("\n\tPlease Enter AWS Access Key: ") if len(self.aws_access_key) < 20: self.show_step_inner_error("\n\t" + K.INVALID_KEY) raise Exception("Invalid AWS Access Key") Settings.set('AWS_ACCESS_KEY', self.aws_access_key) else: self.aws_access_key = settings_access_key
def __init__(self, args): args.append((K.CATEGORY_FIELD_NAME, "deploy")) args.append((K.CATEGORY_FIELD_NAME, "roles")) args.append((K.CATEGORY_FIELD_NAME, "batch-ecr")) args.append((K.CATEGORY_FIELD_NAME, "batch-job")) args.append((K.CATEGORY_FIELD_NAME, "submit-job")) args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job")) args.append((K.CATEGORY_FIELD_NAME, "upload_tf")) self.need_complete_install = self._need_complete_installation() Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True) super().__init__(args)
def run(self, sys_args): self.show_loading_messsage() Settings.set('running_command', ' '.join(sys_args)) try: SysLog().debug_started(Settings.running_command) if self.do_pre_requisite_check(): command_class_instance = self.get_command_class_instance( sys_args) # Get the command list and optional commands self.execute(command_class_instance) except Exception as e: self.show_step_inner_error( "Error occured, please check error log for more details") SysLog().write_error_log(str(e))
def read_aws_secret_key(self): """Read AWS secret key from user if it is not already set in settings""" settings_secret_key = getattr(Settings, 'AWS_SECRET_KEY', None) if settings_secret_key is None or settings_secret_key == '': self.aws_secret_key = input("\n\t%s" % K.AWS_SECRET_KEY_INPUT) if len(self.aws_secret_key) < 25: self.show_step_inner_error("\n\t" + K.INVALID_KEY) raise Exception(K.INVALID_KEY) Settings.set('AWS_SECRET_KEY', self.aws_secret_key) else: self.aws_secret_key = settings_secret_key
def read_aws_session_token(self): """Read AWS session token from user if it is not already set in settings""" settings_session_token = getattr(Settings, 'AWS_SESSION_TOKEN', None) if settings_session_token is None or settings_session_token == '': self.aws_session_token = input("\n\t%s" % K.AWS_SESSION_TOKEN_INPUT) if len(self.aws_session_token) < 25: self.show_step_inner_error("\n\t" + K.INVALID_KEY) raise Exception(K.INVALID_KEY) Settings.set('AWS_SESSION_TOKEN', self.aws_session_token) else: self.aws_session_token = settings_session_token
def validate_system_storage_config(self): """ Validate the system configuration requirements Minimum Storage requirements Validation Returns: valid or not_valid (str): Configured string for valid and not valid conditions """ self.error_message = None statvfs = os.statvfs(os.getcwd()) available_disk_space = (statvfs.f_frsize * statvfs.f_bavail ) / (2 ** 30) if not available_disk_space >= Settings.get('MINIMUM_STORAGE', None): self.error_message = K.INVALID_SYSTEM_STORAGE_CONFIG + str(Settings.get('MINIMUM_STORAGE', None)) return K.NOT_VALID return K.VALID
def validate_system_ram_config(self): """ Validate the system configuration requirements Minimum RAM requirements Validation Returns: valid or not_valid (str): Configured string for valid and not valid conditions """ self.error_message = None system_mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') system_mem_gib = system_mem_bytes / (1024. ** 3) if not system_mem_gib > Settings.get('MINIMUM_RAM', None): self.error_message = K.INVALID_SYSTEM_RAM_CONFIG + str(Settings.get('MINIMUM_RAM', None)) return K.NOT_VALID return K.VALID
def _get_printable_abs_url(self, dns_name): """ This function returns the absolute URL of the domain ie. with http/https Args: dns_name (str): Loadbalancer dns name Returns: url (str): abs url of pacbot """ pacbot_domain = Settings.get('PACBOT_DOMAIN', None) pacbot_domain = pacbot_domain if pacbot_domain else dns_name return "%s://%s" % (Settings.get('ALB_PROTOCOL', "HTTP").lower(), pacbot_domain)
def read_aws_region(self): """Read AWS region from user if it is not already set in settings""" settings_region = getattr(Settings, 'AWS_REGION', None) if settings_region is None or settings_region == '': if self.silent_install: self.show_step_inner_error(K.AWS_REGION_NOT_SUPPLIED) raise Exception(K.AWS_REGION_NOT_SUPPLIED) aws_region = input("\n\t%s" % K.AWS_REGION_INPUT) else: aws_region = settings_region Settings.set('AWS_REGION', aws_region) return aws_region
def init_configure(self): """Pass.""" # self.log.info('start init configure file') if not utils.checkFileExists(utils.getenv('COWRY_CONFIG')): src = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'cowry.conf.default') dst = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'cowry.conf') utils.copyfile(src, dst) print('Not find default configure file, copy default configure to use') self.settings = Settings() # set default certificates folders values if not self.settings.certificates.certdirs: setDefaultCertDirs = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'certs') self.settings._set(('certificates', 'certdirs', setDefaultCertDirs))
def validate_vpc_and_cidr_blocks(self): """ Check the VPC is correct and the CIDR block provided is also correct Returns: valid or not_valid (str): Configured string for valid and not valid conditions """ self.error_message = None if Settings.get('VPC', None): vpc_ids = [Settings.VPC['ID']] cidr_blocks = Settings.VPC['CIDR_BLOCKS'] try: vpcs = vpc.get_vpc_details( vpc_ids, Settings.AWS_AUTH_CRED ) except Exception as e: self.error_message = str(e) + "\n\t" + K.CORRECT_VPC_MSG return K.NOT_VALID valid_cidr_blocks = [vpc['CidrBlock'] for vpc in vpcs] if not set(cidr_blocks).issubset(set(valid_cidr_blocks)): self.error_message = K.INVALID_CIDR + "\n\t" + K.CORRECT_VPC_MSG return K.NOT_VALID return K.VALID
#!/usr/bin/env python from core.config import Settings from core.server import server from optparse import OptionParser __author__ = "Jeremy Subtil" __email__ = "*****@*****.**" parser = OptionParser() parser.add_option("-c", "--config", action="store", type="string", dest="config_filename", default="config.cfg", help="the configuration file to use") (options, args) = parser.parse_args() settings = Settings() settings.loadFromFile(options.config_filename) settings.loadFromArgs(options) server.configure(settings) server.run()