def setUpClass(cls): from consoleme.celery_tasks.celery_tasks import ( cache_roles_across_accounts, cache_roles_for_account, ) from consoleme.config.config import CONFIG old_config = copy.deepcopy(CONFIG.config) CONFIG.config = { **CONFIG.config, "aws": { **CONFIG.config.get("aws", {}), "iamroles_redis_key": "test_cache_roles_for_account", }, "cache_roles_across_accounts": { "all_roles_combined": { "s3": { "file": "TestCloudCredentialAuthorizationMapping.json.gz", } } }, } cache_roles_for_account("123456789012") cache_roles_across_accounts() CONFIG.config = old_config
def setUpClass(cls): from consoleme.celery_tasks.celery_tasks import ( cache_roles_across_accounts, cache_roles_for_account, ) cache_roles_for_account("123456789012") cache_roles_across_accounts()
def populate_caches( redis, user_iam_role, iam_sync_roles, dummy_users_data, dummy_requests_data, policy_requests_table, iamrole_table, create_default_resources, s3, sns, sqs, iam, www_user, parliament, ): from asgiref.sync import async_to_sync from consoleme.celery_tasks import celery_tasks as celery from consoleme.lib.account_indexers import get_account_id_to_name_mapping from consoleme_default_plugins.plugins.celery_tasks import ( celery_tasks as default_celery_tasks, ) celery.cache_cloud_account_mapping() accounts_d = async_to_sync(get_account_id_to_name_mapping)() default_celery_tasks.cache_application_information() for account_id in accounts_d.keys(): celery.cache_roles_for_account(account_id) celery.cache_s3_buckets_for_account(account_id) celery.cache_sns_topics_for_account(account_id) celery.cache_sqs_queues_for_account(account_id) celery.cache_managed_policies_for_account(account_id) # celery.cache_resources_from_aws_config_for_account(account_id) # No select_resource_config in moto yet # Running cache_roles_across_accounts ensures that all of the pre-existing roles in our role cache are stored in # (mock) S3 celery.cache_roles_across_accounts() celery.cache_policies_table_details() celery.cache_policy_requests() celery.cache_credential_authorization_mapping()
def create_default_resources(s3, iam, redis, iam_sync_roles, iamrole_table): from asgiref.sync import async_to_sync from consoleme.config import config from consoleme.lib.cache import store_json_results_in_redis_and_s3 global all_roles buckets = [config.get("consoleme_s3_bucket")] for bucket in buckets: s3.create_bucket(Bucket=bucket) if all_roles: async_to_sync(store_json_results_in_redis_and_s3)( all_roles, s3_bucket=config.get( "cache_roles_across_accounts.all_roles_combined.s3.bucket"), s3_key=config.get( "cache_roles_across_accounts.all_roles_combined.s3.file"), ) return from consoleme.celery_tasks.celery_tasks import cache_roles_for_account from consoleme.lib.account_indexers import get_account_id_to_name_mapping from consoleme.lib.redis import RedisHandler red = RedisHandler().redis_sync() accounts_d = async_to_sync(get_account_id_to_name_mapping)() for account_id in accounts_d.keys(): cache_roles_for_account(account_id) cache_key = config.get("aws.iamroles_redis_key", "IAM_ROLE_CACHE") all_roles = red.hgetall(cache_key) async_to_sync(store_json_results_in_redis_and_s3)( all_roles, s3_bucket=config.get( "cache_roles_across_accounts.all_roles_combined.s3.bucket"), s3_key=config.get( "cache_roles_across_accounts.all_roles_combined.s3.file"), )
# `celery -A consoleme.celery_tasks.celery_tasks worker -l DEBUG -B -E --concurrency=8` celery.cache_roles_across_accounts() celery.cache_s3_buckets_across_accounts() celery.cache_sns_topics_across_accounts() celery.cache_sqs_queues_across_accounts() celery.cache_managed_policies_across_accounts() default_celery_tasks.cache_application_information() celery.cache_resources_from_aws_config_across_accounts() celery.cache_policies_table_details.apply_async(countdown=180) celery.cache_policy_requests() celery.cache_credential_authorization_mapping.apply_async(countdown=180) else: celery.cache_cloud_account_mapping() accounts_d = async_to_sync(get_account_id_to_name_mapping)() default_celery_tasks.cache_application_information() for account_id in accounts_d.keys(): celery.cache_roles_for_account(account_id) celery.cache_s3_buckets_for_account(account_id) celery.cache_sns_topics_for_account(account_id) celery.cache_sqs_queues_for_account(account_id) celery.cache_managed_policies_for_account(account_id) celery.cache_resources_from_aws_config_for_account(account_id) celery.cache_policies_table_details() celery.cache_policy_requests() celery.cache_credential_authorization_mapping() print("Done caching redis data")