def populate_caches(
redis,
user_iam_role,
iam_sync_principals,
dummy_users_data,
dummy_requests_data,
policy_requests_table,
iamrole_table,
create_default_resources,
s3,
sns,
sqs,
iam,
www_user,
parliament,
):
from asgiref.sync import async_to_sync
from consoleme.celery_tasks import celery_tasks as celery
from consoleme.lib.account_indexers import get_account_id_to_name_mapping
from consoleme_default_plugins.plugins.celery_tasks import (
celery_tasks as default_celery_tasks,
)
celery.cache_cloud_account_mapping()
accounts_d = async_to_sync(get_account_id_to_name_mapping)()
default_celery_tasks.cache_application_information()
for account_id in accounts_d.keys():
celery.cache_iam_resources_for_account(account_id)
celery.cache_s3_buckets_for_account(account_id)
celery.cache_sns_topics_for_account(account_id)
celery.cache_sqs_queues_for_account(account_id)
celery.cache_managed_policies_for_account(account_id)
# celery.cache_resources_from_aws_config_for_account(account_id) # No select_resource_config in moto yet
# Running cache_iam_resources_across_accounts ensures that all of the pre-existing roles in our
# role cache are stored in (mock) S3
celery.cache_iam_resources_across_accounts()
celery.cache_policies_table_details()
celery.cache_policy_requests()
celery.cache_credential_authorization_mapping()
config.CONFIG.load_config_from_dynamo()
if args.use_celery:
# Initialize Redis locally. If use_celery is set to `True`, you must be running a celery beat and worker. You can
# run this locally with the following command:
# `celery -A consoleme.celery_tasks.celery_tasks worker -l DEBUG -B -E --concurrency=8`
celery.cache_iam_resources_across_accounts()
celery.cache_s3_buckets_across_accounts()
celery.cache_sns_topics_across_accounts()
celery.cache_sqs_queues_across_accounts()
celery.cache_managed_policies_across_accounts()
default_celery_tasks.cache_application_information()
celery.cache_resources_from_aws_config_across_accounts()
celery.cache_policies_table_details.apply_async(countdown=180)
celery.cache_policy_requests()
celery.cache_credential_authorization_mapping.apply_async(countdown=180)
else:
celery.cache_cloud_account_mapping()
accounts_d = async_to_sync(get_account_id_to_name_mapping)(force_sync=True)
default_celery_tasks.cache_application_information()
executor = ThreadPoolExecutor(max_workers=os.cpu_count())
futures = []
for account_id in accounts_d.keys():
futures.extend(
[
executor.submit(celery.cache_iam_resources_for_account, account_id),
executor.submit(celery.cache_s3_buckets_for_account, account_id),
executor.submit(celery.cache_sns_topics_for_account, account_id),
executor.submit(celery.cache_sqs_queues_for_account, account_id),