def read_aws_region(self):
settings_region = getattr(Settings, 'AWS_REGION', None)
if settings_region is None or settings_region == '':
self.aws_region = input("\n\tPlease Enter AWS Region: ")
Settings.set('AWS_REGION', self.aws_region)
else:
self.aws_region = settings_region
def __init__(self, args):
args.append((K.CATEGORY_FIELD_NAME, "deploy"))
args.append((K.CATEGORY_FIELD_NAME, "roles"))
self.need_complete_install = self._need_complete_installation()
Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
super().__init__(args)
class ApplicationLoadBalancer(LoadBalancerResource):
name = ""
internal = Settings.get('MAKE_ALB_INTERNAL', True)
load_balancer_type = "application"
security_groups = [InfraSecurityGroupResource.get_output_attr('id')]
subnets = Settings.get('VPC')['SUBNETS']
OUTPUT_LIST = ['dns_name']
@classmethod
def get_http_url(cls):
return "http://%s" % cls.get_output_attr('dns_name')
@classmethod
def get_api_base_url(cls):
return "http://%s/api" % cls.get_output_attr('dns_name')
@classmethod
def get_api_version_url(cls, service):
version_url = cls.get_api_server_url(service)
return version_url if service == "auth" else version_url + "/v1"
@classmethod
def get_api_server_url(cls, service):
return "%s/%s" % (cls.get_api_base_url(), service)
def render_output(self, outputs):
if self.resource_in_tf_output(outputs):
return {
'Pacbot Domain': outputs[self.get_resource_id()]['dns_name'],
'Admin': Settings.PACBOT_LOGIN_CREDENTIALS['Admin'],
'User': Settings.PACBOT_LOGIN_CREDENTIALS['User']
}
def validate_subnet_ids(self):
self.error_message = None
if Settings.get('VPC', None):
vpc_ids = [Settings.VPC['ID']]
if Settings.VPC.get('SUBNETS', None):
if Settings.get('REQUIRE_SUBNETS_ON_DIFFERENT_ZONE', False):
subnet_ids = Settings.VPC['SUBNETS']
try:
valid_subnets = vpc.get_vpc_subnets(
Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY,
Settings.AWS_REGION, vpc_ids)
except Exception as e:
self.error_message = str(
e) + "\n\t" + K.CORRECT_VPC_MSG
return K.NOT_VALID
current_subnets = [
subnet for subnet in valid_subnets
if subnet['SubnetId'] in subnet_ids
]
if len(current_subnets) != len(subnet_ids):
self.error_message = K.INVALID_SUBNETS
return K.NOT_VALID
if len(
set([
subnet['AvailabilityZone']
for subnet in current_subnets
])) < 2:
self.error_message = K.INVALID_SUBNET_ZONES
return K.NOT_VALID
return K.VALID
def __init__(self, args):
Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
args.append((K.CATEGORY_FIELD_NAME, "deploy"))
args.append((K.CATEGORY_FIELD_NAME, "upload_tf"))
self.destroy_resource_tags_list = [
v for (k, v) in args if k == self.category_field_name
]
args.append((K.CATEGORY_FIELD_NAME, "deploy"))
args.append((K.CATEGORY_FIELD_NAME, "roles"))
args.append((K.CATEGORY_FIELD_NAME, "all_read_role"))
args.append((K.CATEGORY_FIELD_NAME, "batch-ecr"))
args.append((K.CATEGORY_FIELD_NAME, "batch-job"))
args.append((K.CATEGORY_FIELD_NAME, "submit-job"))
args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job"))
args.append((K.CATEGORY_FIELD_NAME, "upload_tf"))
self.reinstall_resource_tags_list = [
v for (k, v) in args if k == self.category_field_name
]
self.need_complete_install = self._need_complete_installation()
self.dry_run = True if any([x[1] for x in args
if x[0] == "dry-run"]) else self.dry_run
self.silent_install = True if any(
[x[1] for x in args if x[0] == "silent"]) else self.silent_install
def load_aws_account_details(self):
"""Find AWS Account ID from the credentials given"""
caller_identity = get_aws_caller_identity(self.AWS_AUTH_CRED)
Settings.set('AWS_ACCOUNT_ID', caller_identity.get('Account'))
Settings.set('CALLER_ARN', caller_identity.get('Arn'))
self.AWS_ACCOUNT_ID = caller_identity.get('Account')
self.CALLER_ARN = caller_identity.get('Arn')
def load_aws_account_id(self):
"""Find AWS Account ID from the credentials given"""
aws_account_id = get_user_account_id(Settings.AWS_ACCESS_KEY,
Settings.AWS_SECRET_KEY,
Settings.AWS_SESSION_TOKEN)
Settings.set('AWS_ACCOUNT_ID', aws_account_id)
self.aws_account_id = aws_account_id
def get_pacbot_domain_url(cls):
pacbot_domain = Settings.get('PACBOT_DOMAIN', None)
pacbot_domain = pacbot_domain if pacbot_domain else cls.get_output_attr(
'dns_name')
return "%s://%s" % (Settings.get('ALB_PROTOCOL',
"HTTP").lower(), pacbot_domain)
def read_aws_region(self):
settings_region = getattr(Settings, 'AWS_REGION', None)
if settings_region is None or settings_region == '':
self.aws_region = input("\n\t%s" % K.AWS_REGION_INPUT)
Settings.set('AWS_REGION', self.aws_region)
else:
self.aws_region = settings_region
def validate(self):
self.show_step_heading(K.SETTINGS_CHECK_STARTED)
status = self.validate_vpc_and_cidr_blocks()
self.show_step_inner_messaage(K.VPC_CHECK_STARTED, status, self.error_message)
if status != K.VALID:
return False
status = self.validate_subnet_ids()
self.show_step_inner_messaage(K.SUBNETS_CHECK_STARTED, status, self.error_message)
if status != K.VALID:
return False
status = self.validate_policies()
if Settings.get('MINIMUM_RAM', None):
status = self.validate_system_ram_config()
self.show_step_inner_messaage(K.SYSTEM_RAM_CONFIG_CHECK_STARTED, status, self.error_message)
if status != K.VALID:
return False
if Settings.get('MINIMUM_STORAGE', None):
status = self.validate_system_storage_config()
self.show_step_inner_messaage(K.SYSTEM_STORAGE_CONFIG_CHECK_STARTED, status, self.error_message)
if status != K.VALID:
return False
return status
class InfraSecurityGroupResource(SecurityGroupResource):
name = ""
vpc_id = Settings.get('VPC')['ID']
ingress = [{
'from_port': 0,
'to_port': 0,
'protocol': "-1",
'cidr_blocks': Settings.get('VPC')['CIDR_BLOCKS'],
'ipv6_cidr_blocks': [],
'prefix_list_ids': [],
'description': "",
'self': False,
'security_groups': []
}]
egress = [{
'from_port': 0,
'to_port': 0,
'protocol': "-1",
'cidr_blocks': ["0.0.0.0/0"],
'ipv6_cidr_blocks': [],
'prefix_list_ids': [],
'description': "",
'self': False,
'security_groups': []
}]
def validate_subnet_ids(self):
"""
Check the subnets provided are present under the given VPC or not
Returns:
valid or not_valid (str): Configured string for valid and not valid conditions
"""
self.error_message = None
if Settings.get('VPC', None):
vpc_ids = [Settings.VPC['ID']]
if Settings.VPC.get('SUBNETS', None):
if Settings.get('REQUIRE_SUBNETS_ON_DIFFERENT_ZONE', False):
subnet_ids = Settings.VPC['SUBNETS']
try:
valid_subnets = vpc.get_vpc_subnets(
vpc_ids,
Settings.AWS_AUTH_CRED
)
except Exception as e:
self.error_message = str(e) + "\n\t" + K.CORRECT_VPC_MSG
return K.NOT_VALID
current_subnets = [subnet for subnet in valid_subnets if subnet['SubnetId'] in subnet_ids]
if len(current_subnets) != len(subnet_ids):
self.error_message = K.INVALID_SUBNETS
return K.NOT_VALID
if len(set([subnet['AvailabilityZone'] for subnet in current_subnets])) < 2:
self.error_message = K.INVALID_SUBNET_ZONES
return K.NOT_VALID
return K.VALID
def read_aws_region(self):
"""Read AWS region from user if it is not already set in settings"""
settings_region = getattr(Settings, 'AWS_REGION', None)
if settings_region is None or settings_region == '':
self.aws_region = input("\n\t%s" % K.AWS_REGION_INPUT)
Settings.set('AWS_REGION', self.aws_region)
else:
self.aws_region = settings_region
def load_settings(self, config_path):
"""
Load all the main and local configurations into the system
Args:
config_path (str): This is the path to the main configuration/settings file
"""
Settings.load_setings(config_path)
def show_step_heading(self, heading, write_log=True):
if write_log:
SysLog().write_debug_log(heading)
step_count_num = Settings.get('step_count_num', 1)
print(
self._get_heading_message_in_color(
"\nStep %s: %s" % (str(step_count_num), heading),
self.BCYAN_ANSI))
step_count_num = Settings.set('step_count_num', step_count_num + 1)
class RuleEngineBatchJobEnv(BatchComputeEnvironmentResource):
compute_environment_name = ""
instance_role = ECSRoleInstanceProfile.get_output_attr('arn')
instance_type = [Settings.get('BATCH_INSTANCE_TYPE', "m4.xlarge")]
max_vcpus = 256
min_vcpus = 0
desired_vcpus = 0
ec2_key_pair = ""
resource_type = "EC2"
security_group_ids = [InfraSecurityGroupResource.get_output_attr('id')]
subnets = Settings.get('VPC')['SUBNETS']
env_type = "MANAGED"
service_role = BatchRole.get_output_attr('arn')
compute_resources_tags = get_all_resource_tags()
DEPENDS_ON = [
BatchIAMRolePolicyAttach
] # This is required otherwise policy would be dettached from Batchrole
def pre_terraform_apply(self):
ec2_client = get_ec2_client(self.input.AWS_AUTH_CRED)
ec2_key_pair = self.get_input_attr('ec2_key_pair')
try:
key_obj = ec2_client.create_key_pair(KeyName=ec2_key_pair)
with open(os.path.join(Settings.OUTPUT_DIR, ec2_key_pair + ".pem"),
"w") as keyfile:
keyfile.write(key_obj['KeyMaterial'])
except Exception as e:
pass
def check_batch_jobs_running(self):
envs = get_compute_environments(
[self.get_input_attr('compute_environment_name')],
self.input.AWS_AUTH_CRED)
if not len(envs):
return
if envs[0]['computeResources']['desiredvCpus'] > int(
self.get_input_attr('desired_vcpus')):
return True
def pre_generate_terraform(self):
warn_msg = "Batch Jobs are running, please try after it gets completed and desired CPUs comes to 0."
if self.check_batch_jobs_running():
message = "\n\t ** %s **\n" % warn_msg
print(MsgMixin.BERROR_ANSI + message + MsgMixin.RESET_ANSI)
sys.exit()
def post_terraform_destroy(self):
ec2_client = get_ec2_client(self.input.AWS_AUTH_CRED)
ec2_key_pair = self.get_input_attr('ec2_key_pair')
try:
key_obj = ec2_client.delete_key_pair(KeyName=ec2_key_pair)
except Exception as e:
print(ec2_key_pair +
" Not able to delete Key Pair. Error: %s" % str(e))
def read_aws_access_key(self):
settings_access_key = getattr(Settings, 'AWS_ACCESS_KEY', None)
if settings_access_key is None or settings_access_key == '':
self.aws_access_key = input("\n\t%s" % K.AWS_ACCESS_KEY_INPUT)
if len(self.aws_access_key) < 20:
self.show_step_inner_error("\n\t" + K.INVALID_KEY)
raise Exception(K.INVALID_KEY)
Settings.set('AWS_ACCESS_KEY', self.aws_access_key)
else:
self.aws_access_key = settings_access_key
def __init__(self, args):
args.append((K.CATEGORY_FIELD_NAME, "deploy"))
args.append((K.CATEGORY_FIELD_NAME, "batch-ecr"))
args.append((K.CATEGORY_FIELD_NAME, "batch-job"))
args.append((K.CATEGORY_FIELD_NAME, "submit-job"))
args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job"))
args.append((K.CATEGORY_FIELD_NAME, "upload_tf"))
Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
super().__init__(args)
def __init__(self, args):
# args.append((K.CATEGORY_FIELD_NAME, "datastore"))
# tf_outputs = PyTerraform.load_terraform_output_from_json_file()
# role_file = os.path.join(Settings.TERRAFORM_DIR, "iam_all_read_role_AllReadRole.tf")
# if not tf_outputs.get(AllReadRole.get_resource_id(), False):
# args.append((K.CATEGORY_FIELD_NAME, "all_read_role"))
# args.append((K.CATEGORY_FIELD_NAME, "ecs_role"))
Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
super().__init__(args)
def read_aws_access_key(self):
settings_access_key = getattr(Settings, 'AWS_ACCESS_KEY', None)
if settings_access_key is None or settings_access_key == '':
self.aws_access_key = input("\n\tPlease Enter AWS Access Key: ")
if len(self.aws_access_key) < 20:
self.show_step_inner_error("\n\t" + K.INVALID_KEY)
raise Exception("Invalid AWS Access Key")
Settings.set('AWS_ACCESS_KEY', self.aws_access_key)
else:
self.aws_access_key = settings_access_key
def __init__(self, args):
args.append((K.CATEGORY_FIELD_NAME, "deploy"))
args.append((K.CATEGORY_FIELD_NAME, "roles"))
args.append((K.CATEGORY_FIELD_NAME, "batch-ecr"))
args.append((K.CATEGORY_FIELD_NAME, "batch-job"))
args.append((K.CATEGORY_FIELD_NAME, "submit-job"))
args.append((K.CATEGORY_FIELD_NAME, "rule-engine-job"))
args.append((K.CATEGORY_FIELD_NAME, "upload_tf"))
self.need_complete_install = self._need_complete_installation()
Settings.set('SKIP_RESOURCE_EXISTENCE_CHECK', True)
super().__init__(args)
def run(self, sys_args):
self.show_loading_messsage()
Settings.set('running_command', ' '.join(sys_args))
try:
SysLog().debug_started(Settings.running_command)
if self.do_pre_requisite_check():
command_class_instance = self.get_command_class_instance(
sys_args) # Get the command list and optional commands
self.execute(command_class_instance)
except Exception as e:
self.show_step_inner_error(
"Error occured, please check error log for more details")
SysLog().write_error_log(str(e))
def read_aws_secret_key(self):
"""Read AWS secret key from user if it is not already set in settings"""
settings_secret_key = getattr(Settings, 'AWS_SECRET_KEY', None)
if settings_secret_key is None or settings_secret_key == '':
self.aws_secret_key = input("\n\t%s" % K.AWS_SECRET_KEY_INPUT)
if len(self.aws_secret_key) < 25:
self.show_step_inner_error("\n\t" + K.INVALID_KEY)
raise Exception(K.INVALID_KEY)
Settings.set('AWS_SECRET_KEY', self.aws_secret_key)
else:
self.aws_secret_key = settings_secret_key
def read_aws_session_token(self):
"""Read AWS session token from user if it is not already set in settings"""
settings_session_token = getattr(Settings, 'AWS_SESSION_TOKEN', None)
if settings_session_token is None or settings_session_token == '':
self.aws_session_token = input("\n\t%s" %
K.AWS_SESSION_TOKEN_INPUT)
if len(self.aws_session_token) < 25:
self.show_step_inner_error("\n\t" + K.INVALID_KEY)
raise Exception(K.INVALID_KEY)
Settings.set('AWS_SESSION_TOKEN', self.aws_session_token)
else:
self.aws_session_token = settings_session_token
def validate_system_storage_config(self):
"""
Validate the system configuration requirements
Minimum Storage requirements Validation
Returns:
valid or not_valid (str): Configured string for valid and not valid conditions
"""
self.error_message = None
statvfs = os.statvfs(os.getcwd())
available_disk_space = (statvfs.f_frsize * statvfs.f_bavail ) / (2 ** 30)
if not available_disk_space >= Settings.get('MINIMUM_STORAGE', None):
self.error_message = K.INVALID_SYSTEM_STORAGE_CONFIG + str(Settings.get('MINIMUM_STORAGE', None))
return K.NOT_VALID
return K.VALID
def validate_system_ram_config(self):
"""
Validate the system configuration requirements
Minimum RAM requirements Validation
Returns:
valid or not_valid (str): Configured string for valid and not valid conditions
"""
self.error_message = None
system_mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
system_mem_gib = system_mem_bytes / (1024. ** 3)
if not system_mem_gib > Settings.get('MINIMUM_RAM', None):
self.error_message = K.INVALID_SYSTEM_RAM_CONFIG + str(Settings.get('MINIMUM_RAM', None))
return K.NOT_VALID
return K.VALID
def _get_printable_abs_url(self, dns_name):
"""
This function returns the absolute URL of the domain ie. with http/https
Args:
dns_name (str): Loadbalancer dns name
Returns:
url (str): abs url of pacbot
"""
pacbot_domain = Settings.get('PACBOT_DOMAIN', None)
pacbot_domain = pacbot_domain if pacbot_domain else dns_name
return "%s://%s" % (Settings.get('ALB_PROTOCOL',
"HTTP").lower(), pacbot_domain)
def read_aws_region(self):
"""Read AWS region from user if it is not already set in settings"""
settings_region = getattr(Settings, 'AWS_REGION', None)
if settings_region is None or settings_region == '':
if self.silent_install:
self.show_step_inner_error(K.AWS_REGION_NOT_SUPPLIED)
raise Exception(K.AWS_REGION_NOT_SUPPLIED)
aws_region = input("\n\t%s" % K.AWS_REGION_INPUT)
else:
aws_region = settings_region
Settings.set('AWS_REGION', aws_region)
return aws_region
def init_configure(self):
"""Pass."""
# self.log.info('start init configure file')
if not utils.checkFileExists(utils.getenv('COWRY_CONFIG')):
src = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'cowry.conf.default')
dst = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'cowry.conf')
utils.copyfile(src, dst)
print('Not find default configure file, copy default configure to use')
self.settings = Settings()
# set default certificates folders values
if not self.settings.certificates.certdirs:
setDefaultCertDirs = utils.joinFilePath(utils.getenv('COWRY_ROOT'), 'certs')
self.settings._set(('certificates', 'certdirs', setDefaultCertDirs))
def validate_vpc_and_cidr_blocks(self):
"""
Check the VPC is correct and the CIDR block provided is also correct
Returns:
valid or not_valid (str): Configured string for valid and not valid conditions
"""
self.error_message = None
if Settings.get('VPC', None):
vpc_ids = [Settings.VPC['ID']]
cidr_blocks = Settings.VPC['CIDR_BLOCKS']
try:
vpcs = vpc.get_vpc_details(
vpc_ids,
Settings.AWS_AUTH_CRED
)
except Exception as e:
self.error_message = str(e) + "\n\t" + K.CORRECT_VPC_MSG
return K.NOT_VALID
valid_cidr_blocks = [vpc['CidrBlock'] for vpc in vpcs]
if not set(cidr_blocks).issubset(set(valid_cidr_blocks)):
self.error_message = K.INVALID_CIDR + "\n\t" + K.CORRECT_VPC_MSG
return K.NOT_VALID
return K.VALID
#!/usr/bin/env python
from core.config import Settings
from core.server import server
from optparse import OptionParser
__author__ = "Jeremy Subtil"
__email__ = "*****@*****.**"
parser = OptionParser()
parser.add_option("-c", "--config", action="store", type="string",
dest="config_filename", default="config.cfg",
help="the configuration file to use")
(options, args) = parser.parse_args()
settings = Settings()
settings.loadFromFile(options.config_filename)
settings.loadFromArgs(options)
server.configure(settings)
server.run()
