class CloudNotificationCollectorCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = CloudNotificationCollectorEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'CloudNotificationCollectorTarget' # Unique identifier
target_input = json.dumps({
'jobName': "aws-cloud-notification-collector",
'jobUuid': "pacman-cloud-notifications-jar-with-dependencies",
'jobType': "jar",
'jobDesc': "Health Notification Collector",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/api/prd/latest"},
{'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()},
{'name': "LOGGING_ES_HOST_NAME", 'value': ESDomain.get_http_url_with_port()},
{'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()},
{'name': "ENVIRONMENT", 'value': "prd"},
{'name': "APP_NAME", 'value': "aws-cloud-notification-collector"},
{'name': "APP_TYPE", 'value': "etl"},
{'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')},
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "conf_src", 'value': "api-prd,application-prd"},
]
})
class RecommendationsCollectorCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = RecommendationsCollectorEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'RecommendationsCollectorTarget' # Unique identifier
target_input = json.dumps({
'jobName': "aws-recommendations-collector",
'jobUuid': "recommendation-enricher-jar-with-dependencies",
'jobType': "jar",
'jobDesc': "Index trusted advisor checks as recommendations",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,recommendation-enricher/prd/latest"},
{'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()},
{'name': "LOGGING_ES_HOST_NAME", 'value': ESDomain.get_http_url_with_port()},
{'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()},
{'name': "ENVIRONMENT", 'value': "prd"},
{'name': "APP_NAME", 'value': "aws-recommendations-collector"},
{'name': "APP_TYPE", 'value': "etl"},
{'name': "HEIMDALL_URI", 'value': ESDomain.get_http_url_with_port()},
{'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')},
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacbot"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
]
})
class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = DataShipperEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'DataShipperTarger' # Unique identifier
target_input = json.dumps({
'jobName': "aws-redshift-es-data-shipper",
'jobUuid': "data-shipper-jar-with-dependencies",
'jobType': "jar",
'jobDesc': "Ship aws data periodically from redshfit to ES",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,data-shipper/prd/latest"},
{'name': "ASSET_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('asset')},
{'name': "CMPL_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('compliance')},
{'name': "AUTH_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('auth')},
{'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4="},
{'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest"}
] + ([{
'name': "VULN_API_URL",
'value': ApplicationLoadBalancer.get_api_version_url('vulnerability')}
] if need_to_deploy_vulnerability_service() else []),
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"},
{'encrypt': False, 'key': "datasource", 'value': "aws"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "apiauthinfo",
'value': "MjJlMTQ5MjItODdkNy00ZWU0LWE0NzAtZGEwYmIxMGQ0NWQzOmNzcldwYzVwN0pGRjR2RVpCa3dHQ0FoNjdrR1FHd1h2NDZxdWc3djVad3RLZw=="}
]
})
class SubmitAndRuleEngineJobDefinition(BatchJobDefinitionResource):
name = 'rule-engine'
jd_type = 'container'
attempts = 2
container_properties = json.dumps({
'command': [
"~/fetch_and_run.sh",
"Ref::executableName",
"Ref::params",
"Ref::jvmMemParams",
"Ref::ruleEngineExecutableName",
"Ref::entryPoint"
],
'image': RuleEngineEcrRepository.get_output_attr('repository_url'),
'memory': 5000,
'vcpus': 1,
'environment': [
{'name': "ES_HOST", 'value': ESDomain.get_http_url_with_port()},
{'name': "BASE_AWS_ACCOUNT", 'value': AwsAccount.get_output_attr('account_id')},
{'name': "ES_URI", 'value': ESDomain.get_http_url_with_port()},
{'name': "HEIMDALL_URI", 'value': ESDomain.get_http_url_with_port()},
{'name': "PACMAN_API_URI", 'value': ApplicationLoadBalancer.get_api_base_url()}
]
})
def post_terraform_destroy(self):
delete_task_definition(
Settings.AWS_ACCESS_KEY,
Settings.AWS_SECRET_KEY,
Settings.AWS_REGION,
self.get_input_attr('name')
)
class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = DataCollectorEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'DataCollectorTarget' # Unique identifier
target_input = json.dumps({
'jobName': "AWS-Data-Collector",
'jobUuid': "pacman-aws-inventory-jar-with-dependencies",
'jobType': "jar",
'jobDesc': "AWS-Data-Collection",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,inventory/prd/latest"},
{'name': "CONFIG_CREDENTIALS", 'value': "dXNlcjpwYWNtYW4="},
{'name': "CONFIG_SERVICE_URL", 'value': ApplicationLoadBalancer.get_http_url() + "/api/config/rule/prd/latest"}
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "accountinfo", 'value': AwsAccount.get_output_attr('account_id')},
# {'encrypt': False, 'key': "base-account", 'value': AwsAccount.get_output_attr('account_id')},
# {'encrypt': False, 'key': "discovery-role", 'value': BaseRole.get_output_attr('name')},
# {'encrypt': False, 'key': "s3", 'value': BucketStorage.get_output_attr('bucket')},
# {'encrypt': False, 'key': "s3-data", 'value': "inventory"}, # TODO: need to be changed with s3obj class
# {'encrypt': False, 'key': "s3-processed", 'value': "backup"},
# {'encrypt': False, 'key': "s3-role", 'value': BaseRole.get_output_attr('name')},
# {'encrypt': False, 'key': "s3-region", 'value': AwsRegion.get_output_attr('name')},
# {'encrypt': False, 'key': "file-path", 'value': "/home/ec2-user/data"},
# {'encrypt': False, 'key': "base-region", 'value': AwsRegion.get_output_attr('name')}
]
})
class SubmitJobLambdaFunction(LambdaFunctionResource):
function_name = "datacollector"
role = LambdaRole.get_output_attr('arn')
handler = BATCH_JOB_FILE_NAME + ".lambda_handler"
runtime = "python2.7"
s3_bucket = BucketStorage.get_output_attr('bucket')
s3_key = UploadLambdaSubmitJobZipFile.get_output_attr('id')
environment = {
'variables': {
'JOB_QUEUE':
BatchJobsQueue.get_input_attr('name'),
'JOB_DEFINITION':
SubmitAndRuleEngineJobDefinition.get_output_attr('arn'),
'CONFIG_URL':
ApplicationLoadBalancer.get_api_base_url() +
"/config/batch,inventory/prd/latest",
'CONFIG_CREDENTIALS':
"dXNlcjpwYWNtYW4=",
'CONFIG_SERVICE_URL':
ApplicationLoadBalancer.get_http_url() +
"/api/config/rule/prd/latest"
}
}
DEPENDS_ON = [SubmitAndRuleEngineJobDefinition, BatchJobsQueue]
class DataShipperCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = DataShipperEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'DataShipperTarger' # Unique identifier
target_input = json.dumps({
'jobName': "aws-redshift-es-data-shipper",
'jobUuid': "data-shipper-jar-with-dependencies",
'jobType': "jar",
'jobDesc': "Ship aws data periodically from redshfit to ES",
'environmentVariables': [
# {'name': "ES_HOST", 'value': ESDomain.get_output_attr('endpoint')},
# {'name': "RDS_DB_URL", 'value': MySQLDatabase.get_rds_db_url()},
# {'name': "ES_PORT", 'value': "80"},
# {'name': "STAT_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('statistics')},
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,data-shipper/prd/latest"},
{'name': "ASSET_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('asset')},
{'name': "CMPL_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('compliance')},
{'name': "AUTH_API_URL", 'value': ApplicationLoadBalancer.get_api_version_url('auth')},
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile"},
{'encrypt': False, 'key': "datasource", 'value': "aws"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "apiauthinfo",
'value': "MjJlMTQ5MjItODdkNy00ZWU0LWE0NzAtZGEwYmIxMGQ0NWQzOmNzcldwYzVwN0pGRjR2RVpCa3dHQ0FoNjdrR1FHd1h2NDZxdWc3djVad3RLZw=="}
]
})
class SubmitAndRuleEngineJobDefinition(BatchJobDefinitionResource):
name = 'rule-engine'
jd_type = 'container'
attempts = 2
container_properties = json.dumps({
'command': [
"~/fetch_and_run.sh", "Ref::executableName", "Ref::params",
"Ref::jvmMemParams", "Ref::ruleEngineExecutableName",
"Ref::entryPoint"
],
'image':
RuleEngineEcrRepository.get_output_attr('repository_url'),
'memory':
5000,
'vcpus':
1,
'environment': [{
'name': "ES_HOST",
'value': ESDomain.get_http_url_with_port()
}, {
'name': "BASE_AWS_ACCOUNT",
'value': AwsAccount.get_output_attr('account_id')
}, {
'name': "ES_URI",
'value': ESDomain.get_http_url_with_port()
}, {
'name': "HEIMDALL_URI",
'value': ESDomain.get_http_url_with_port()
}, {
'name': "PACMAN_API_URI",
'value': ApplicationLoadBalancer.get_api_base_url()
}, {
'name': "CONFIG_CREDENTIALS",
'value': "dXNlcjpwYWNtYW4="
}, {
'name':
"CONFIG_SERVICE_URL",
'value':
ApplicationLoadBalancer.get_http_url() +
"/api/config/rule/prd/latest"
}]
})
def post_terraform_destroy(self):
deregister_task_definition(Settings.AWS_ACCESS_KEY,
Settings.AWS_SECRET_KEY,
Settings.AWS_REGION,
self.get_input_attr('name'))
def pre_terraform_destroy(self):
compute_env = RuleEngineBatchJobEnv.get_input_attr(
'compute_environment_name')
job_definition = self.get_input_attr('name')
utils.remove_batch_job_related_resources(compute_env, job_definition)
class DataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = DataCollectorEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'DataCollectorTarget' # Unique identifier
target_input = json.dumps({
'jobName':
"AWS-Data-Collector",
'jobUuid':
"pacman-aws-inventory-jar-with-dependencies",
'jobType':
"jar",
'jobDesc':
"AWS-Data-Collection",
'environmentVariables': [{
'name':
"CONFIG_URL",
'value':
ApplicationLoadBalancer.get_api_base_url() +
"/config/batch,inventory/prd/latest"
}, {
'name': "CONFIG_CREDENTIALS",
'value': "dXNlcjpwYWNtYW4="
}, {
'name':
"CONFIG_SERVICE_URL",
'value':
ApplicationLoadBalancer.get_http_url() +
"/api/config/rule/prd/latest"
}],
'params': [
{
'encrypt': False,
'key': "package_hint",
'value': "com.tmobile.cso.pacman"
},
{
'encrypt': False,
'key': "config_creds",
'value': "dXNlcjpwYWNtYW4="
},
{
'encrypt': False,
'key': "accountinfo",
'value': AwsAccount.get_output_attr('account_id')
},
]
})
class QualysAssetDataImporterCloudWatchEventTarget(
CloudWatchEventTargetResource):
rule = QualysAssetDataImporterEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'QualysAssetDataImporterTarget' # Unique identifier
target_input = json.dumps({
'jobName':
"qualys-asset-data-importer",
'jobUuid':
"qualys-asset-data-importer",
'jobType':
"jar",
'jobDesc':
"Qualys Asset Data Importer",
'environmentVariables': [
{
'name':
"CONFIG_URL",
'value':
ApplicationLoadBalancer.get_api_base_url() +
"/config/batch,qualys-enricher/prd/latest"
},
],
'params': [{
'encrypt': False,
'key': "package_hint",
'value': "com.tmobile"
}, {
'encrypt': False,
'key': "config_creds",
'value': "dXNlcjpwYWNtYW4="
}, {
'encrypt': False,
'key': "job_hint",
'value': "qualys"
}, {
'encrypt': False,
'key': "server_type",
'value': "ec2"
}, {
'encrypt': False,
'key': "datasource",
'value': "aws"
}]
})
PROCESS = need_to_deploy_vulnerability_service()
class AzureDataShipperCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = AzureDataShipperEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'AzureDataShipperTarget' # Unique identifier
target_input = json.dumps({
'jobName': "data-shipper-azure",
'jobUuid': "data-shipper-azure",
'jobType': "jar",
'jobDesc': "Ship Azure Data from S3 to PacBot ES",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"},
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.cso.pacman"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "datasource", 'value': "azure"},
{'encrypt': False, 'key': "s3.data", 'value': "azure-inventory"}
]
})
PROCESS = need_to_enable_azure()
class AzureDataCollectorCloudWatchEventTarget(CloudWatchEventTargetResource):
rule = AzureDataCollectorEventRule.get_output_attr('name')
arn = SubmitJobLambdaFunction.get_output_attr('arn')
target_id = 'AzureDataCollectorTarget' # Unique identifier
target_input = json.dumps({
'jobName': "pacbot-azure-discovery",
'jobUuid': "pacbot-azure-discovery",
'jobType': "jar",
'jobDesc': "Collects azure data and upload to S3",
'environmentVariables': [
{'name': "CONFIG_URL", 'value': ApplicationLoadBalancer.get_api_base_url() + "/config/batch,azure-discovery/prd/latest"},
],
'params': [
{'encrypt': False, 'key': "package_hint", 'value': "com.tmobile.pacbot"},
{'encrypt': False, 'key': "file.path", 'value': "/home/ec2-user/azure-data"},
{'encrypt': False, 'key': "config_creds", 'value': "dXNlcjpwYWNtYW4="},
{'encrypt': False, 'key': "tenants", 'value': get_azure_tenants()}
]
})
PROCESS = need_to_enable_azure()