def setUpClass(cls): from consoleme.celery.celery_tasks import ( cache_roles_across_accounts, cache_roles_for_account, ) cache_roles_for_account("123456789012") cache_roles_across_accounts()
def create_default_resources(s3, iam, redis, iam_sync_roles, iamrole_table): from asgiref.sync import async_to_sync from consoleme.config import config from consoleme.lib.cache import store_json_results_in_redis_and_s3 global all_roles buckets = [ config.get("cache_roles_across_accounts.all_roles_combined.s3.bucket") ] for bucket in buckets: s3.create_bucket(Bucket=bucket) if all_roles: async_to_sync(store_json_results_in_redis_and_s3)( all_roles, s3_bucket=config.get( "cache_roles_across_accounts.all_roles_combined.s3.bucket"), s3_key=config.get( "cache_roles_across_accounts.all_roles_combined.s3.file"), ) return from consoleme.celery.celery_tasks import cache_roles_for_account from consoleme.lib.account_indexers import get_account_id_to_name_mapping from consoleme.lib.redis import RedisHandler red = RedisHandler().redis_sync() accounts_d = async_to_sync(get_account_id_to_name_mapping)() for account_id in accounts_d.keys(): cache_roles_for_account(account_id) cache_key = config.get("aws.iamroles_redis_key", "IAM_ROLE_CACHE") all_roles = red.hgetall(cache_key) async_to_sync(store_json_results_in_redis_and_s3)( all_roles, s3_bucket=config.get( "cache_roles_across_accounts.all_roles_combined.s3.bucket"), s3_key=config.get( "cache_roles_across_accounts.all_roles_combined.s3.file"), )
def populate_caches( redis, user_iam_role, iam_sync_roles, dummy_users_data, dummy_requests_data, policy_requests_table, iamrole_table, create_default_resources, s3, sns, sqs, iam, www_user, ): from asgiref.sync import async_to_sync from consoleme.celery import celery_tasks as celery from consoleme.lib.account_indexers import get_account_id_to_name_mapping from consoleme_default_plugins.plugins.celery_tasks import ( celery_tasks as default_celery_tasks, ) celery.cache_cloud_account_mapping() accounts_d = async_to_sync(get_account_id_to_name_mapping)() default_celery_tasks.cache_application_information() for account_id in accounts_d.keys(): celery.cache_roles_for_account(account_id) celery.cache_s3_buckets_for_account(account_id) celery.cache_sns_topics_for_account(account_id) celery.cache_sqs_queues_for_account(account_id) celery.cache_managed_policies_for_account(account_id) # celery.cache_resources_from_aws_config_for_account(account_id) # No select_resource_config in moto yet celery.cache_policies_table_details() celery.cache_policy_requests() celery.cache_credential_authorization_mapping()
# `celery -A consoleme.celery.celery_tasks worker -l DEBUG -B -E --concurrency=8` celery.cache_roles_across_accounts() celery.cache_s3_buckets_across_accounts() celery.cache_sns_topics_across_accounts() celery.cache_sqs_queues_across_accounts() celery.cache_managed_policies_across_accounts() default_celery_tasks.cache_application_information() celery.cache_resources_from_aws_config_across_accounts() celery.cache_policies_table_details.apply_async(countdown=180) celery.cache_policy_requests() celery.cache_credential_authorization_mapping.apply_async(countdown=180) else: celery.cache_cloud_account_mapping() accounts_d = async_to_sync(get_account_id_to_name_mapping)() default_celery_tasks.cache_application_information() for account_id in accounts_d.keys(): celery.cache_roles_for_account(account_id) celery.cache_s3_buckets_for_account(account_id) celery.cache_sns_topics_for_account(account_id) celery.cache_sqs_queues_for_account(account_id) celery.cache_managed_policies_for_account(account_id) celery.cache_resources_from_aws_config_for_account(account_id) celery.cache_policies_table_details() celery.cache_policy_requests() celery.cache_credential_authorization_mapping() print("Done caching redis data")