class CloudNotificationCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): rule = CloudNotificationCollectorEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'CloudNotificationCollectorTarget' # Unique identifier target_input = json.dumps({ 'jobName': "aws-cloud-notification-collector", 'jobUuid': "pacman-cloud-notifications-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "Health Notification Collector", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/api/prd/latest"}, {'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()}, {'name': "LOGGING_ES_HOST_NAME", 'value': ESDomain.get_http_url_with_port()}, {'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()}, {'name': "ENVIRONMENT", 'value': "prd"}, {'name': "APP_NAME", 'value': "aws-cloud-notification-collector"}, {'name': "APP_TYPE", 'value': "etl"}, {'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')}, ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "conf_src", 'value': "api-prd,application-prd"}, ] })
class RecommendationsCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): rule = RecommendationsCollectorEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'RecommendationsCollectorTarget' # Unique identifier target_input = json.dumps({ 'jobName': "aws-recommendations-collector", 'jobUuid': "recommendation-enricher-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "Index trusted advisor checks as recommendations", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,recommendation-enricher/prd/latest"}, {'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()}, {'name': "LOGGING_ES_HOST_NAME", 'value': ESDomain.get_http_url_with_port()}, {'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()}, {'name': "ENVIRONMENT", 'value': "prd"}, {'name': "APP_NAME", 'value': "aws-recommendations-collector"}, {'name': "APP_TYPE", 'value': "etl"}, {'name': "HEIMDALL_URI", 'value': ESDomain.get_http_url_with_port()}, {'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')}, ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacbot"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, ] })
class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): rule = DataShipperEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'DataShipperTarger' # Unique identifier target_input = json.dumps({ 'jobName': "aws-redshift-es-data-shipper", 'jobUuid': "data-shipper-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "Ship aws data periodically from redshfit to ES", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,data-shipper/prd/latest"}, {'name': "ASSET_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('asset')}, {'name': "CMPL_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('compliance')}, {'name': "AUTH_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('auth')}, {'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4="}, {'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest"} ] + ([{ 'name': "VULN_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('vulnerability')} ] if need_to_deploy_vulnerability_service() else []), 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"}, {'encrypt': False, 'key': "datasource", 'value': "aws"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "apiauthinfo", 'value': "MjJlMTQ5MjItODdkNy00ZWU0LWE0NzAtZGEwYmIxMGQ0NWQzOmNzcldwYzVwN0pGRjR2RVpCa3dHQ0FoNjdrR1FHd1h2NDZxdWc3djVad3RLZw=="} ] })
class SubmitAndRuleEngineJobDefinition(BatchJobDefinitionResource): name = 'rule-engine' jd_type = 'container' attempts = 2 container_properties = json.dumps({ 'command': [ "~/fetch_and_run.sh", "Ref::executableName", "Ref::params", "Ref::jvmMemParams", "Ref::ruleEngineExecutableName", "Ref::entryPoint" ], 'image': RuleEngineEcrRepository.get_output_attr('repository_url'), 'memory': 5000, 'vcpus': 1, 'environment': [ {'name': "ES_HOST", 'value': ESDomain.get_http_url_with_port()}, {'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')}, {'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()}, {'name': "HEIMDALL_URI", 'value': ESDomain.get_http_url_with_port()}, {'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()} ] }) def post_terraform_destroy(self): delete_task_definition( Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY, Settings.AWS_REGION, self.get_input_attr('name') )
class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): rule = DataCollectorEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'DataCollectorTarget' # Unique identifier target_input = json.dumps({ 'jobName': "AWS-Data-Collector", 'jobUuid': "pacman-aws-inventory-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "AWS-Data-Collection", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,inventory/prd/latest"}, {'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4="}, {'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest"} ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "accountinfo", 'value': AwsAccount.get_output_attr('account_id')}, # {'encrypt': False, 'key': "base-account", 'value': AwsAccount.get_output_attr('account_id')}, # {'encrypt': False, 'key': "discovery-role", 'value': BaseRole.get_output_attr('name')}, # {'encrypt': False, 'key': "s3", 'value': BucketStorage.get_output_attr('bucket')}, # {'encrypt': False, 'key': "s3-data", 'value': "inventory"}, # TODO: need to be changed with s3obj class # {'encrypt': False, 'key': "s3-processed", 'value': "backup"}, # {'encrypt': False, 'key': "s3-role", 'value': BaseRole.get_output_attr('name')}, # {'encrypt': False, 'key': "s3-region", 'value': AwsRegion.get_output_attr('name')}, # {'encrypt': False, 'key': "file-path", 'value': "/home/ec2-user/data"}, # {'encrypt': False, 'key': "base-region", 'value': AwsRegion.get_output_attr('name')} ] })
class SubmitJobLambdaFunction(LambdaFunctionResource): function_name = "datacollector" role = LambdaRole.get_output_attr('arn') handler = BATCH_JOB_FILE_NAME + ".lambda_handler" runtime = "python2.7" s3_bucket = BucketStorage.get_output_attr('bucket') s3_key = UploadLambdaSubmitJobZipFile.get_output_attr('id') environment = { 'variables': { 'JOB_QUEUE': BatchJobsQueue.get_input_attr('name'), 'JOB_DEFINITION': SubmitAndRuleEngineJobDefinition.get_output_attr('arn'), 'CONFIG_URL': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,inventory/prd/latest", 'CONFIG_CREDENTIALS': "dXNlcjpwYWNtYW4=", 'CONFIG_SERVICE_URL': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest" } } DEPENDS_ON = [SubmitAndRuleEngineJobDefinition, BatchJobsQueue]
class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): rule = DataShipperEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'DataShipperTarger' # Unique identifier target_input = json.dumps({ 'jobName': "aws-redshift-es-data-shipper", 'jobUuid': "data-shipper-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "Ship aws data periodically from redshfit to ES", 'environmentVariables': [ # {'name': "ES_HOST", 'value': ESDomain.get_output_attr('endpoint')}, # {'name': "RDS_DB_URL", 'value': MySQLDatabase.get_rds_db_url()}, # {'name': "ES_PORT", 'value': "80"}, # {'name': "STAT_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('statistics')}, {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,data-shipper/prd/latest"}, {'name': "ASSET_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('asset')}, {'name': "CMPL_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('compliance')}, {'name': "AUTH_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('auth')}, ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"}, {'encrypt': False, 'key': "datasource", 'value': "aws"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "apiauthinfo", 'value': "MjJlMTQ5MjItODdkNy00ZWU0LWE0NzAtZGEwYmIxMGQ0NWQzOmNzcldwYzVwN0pGRjR2RVpCa3dHQ0FoNjdrR1FHd1h2NDZxdWc3djVad3RLZw=="} ] })
class SubmitAndRuleEngineJobDefinition(BatchJobDefinitionResource): name = 'rule-engine' jd_type = 'container' attempts = 2 container_properties = json.dumps({ 'command': [ "~/fetch_and_run.sh", "Ref::executableName", "Ref::params", "Ref::jvmMemParams", "Ref::ruleEngineExecutableName", "Ref::entryPoint" ], 'image': RuleEngineEcrRepository.get_output_attr('repository_url'), 'memory': 5000, 'vcpus': 1, 'environment': [{ 'name': "ES_HOST", 'value': ESDomain.get_http_url_with_port() }, { 'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id') }, { 'name': "ES_URI", 'value': ESDomain.get_http_url_with_port() }, { 'name': "HEIMDALL_URI", 'value': ESDomain.get_http_url_with_port() }, { 'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url() }, { 'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4=" }, { 'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest" }] }) def post_terraform_destroy(self): deregister_task_definition(Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY, Settings.AWS_REGION, self.get_input_attr('name')) def pre_terraform_destroy(self): compute_env = RuleEngineBatchJobEnv.get_input_attr( 'compute_environment_name') job_definition = self.get_input_attr('name') utils.remove_batch_job_related_resources(compute_env, job_definition)
class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): rule = DataCollectorEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'DataCollectorTarget' # Unique identifier target_input = json.dumps({ 'jobName': "AWS-Data-Collector", 'jobUuid': "pacman-aws-inventory-jar-with-dependencies", 'jobType': "jar", 'jobDesc': "AWS-Data-Collection", 'environmentVariables': [{ 'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,inventory/prd/latest" }, { 'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4=" }, { 'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest" }], 'params': [ { 'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman" }, { 'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4=" }, { 'encrypt': False, 'key': "accountinfo", 'value': AwsAccount.get_output_attr('account_id') }, ] })
class QualysAssetDataImporterCloudWatchEventTarget( CloudWatchEventTargetResource): rule = QualysAssetDataImporterEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'QualysAssetDataImporterTarget' # Unique identifier target_input = json.dumps({ 'jobName': "qualys-asset-data-importer", 'jobUuid': "qualys-asset-data-importer", 'jobType': "jar", 'jobDesc': "Qualys Asset Data Importer", 'environmentVariables': [ { 'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,qualys-enricher/prd/latest" }, ], 'params': [{ 'encrypt': False, 'key': "package_hint", 'value': "com.tmobile" }, { 'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4=" }, { 'encrypt': False, 'key': "job_hint", 'value': "qualys" }, { 'encrypt': False, 'key': "server_type", 'value': "ec2" }, { 'encrypt': False, 'key': "datasource", 'value': "aws" }] }) PROCESS = need_to_deploy_vulnerability_service()
class AzureDataShipperCloudWatchEventTarget(CloudWatchEventTargetResource): rule = AzureDataShipperEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'AzureDataShipperTarget' # Unique identifier target_input = json.dumps({ 'jobName': "data-shipper-azure", 'jobUuid': "data-shipper-azure", 'jobType': "jar", 'jobDesc': "Ship Azure Data from S3 to PacBot ES", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"}, ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "datasource", 'value': "azure"}, {'encrypt': False, 'key': "s3.data", 'value': "azure-inventory"} ] }) PROCESS = need_to_enable_azure()
class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource): rule = AzureDataCollectorEventRule.get_output_attr('name') arn = SubmitJobLambdaFunction.get_output_attr('arn') target_id = 'AzureDataCollectorTarget' # Unique identifier target_input = json.dumps({ 'jobName': "pacbot-azure-discovery", 'jobUuid': "pacbot-azure-discovery", 'jobType': "jar", 'jobDesc': "Collects azure data and upload to S3", 'environmentVariables': [ {'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"}, ], 'params': [ {'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.pacbot"}, {'encrypt': False, 'key': "file.path", 'value': "/home/ec2-user/azure-data"}, {'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="}, {'encrypt': False, 'key': "tenants", 'value': get_azure_tenants()} ] }) PROCESS = need_to_enable_azure()