def setUpClass(cls):
from consoleme.celery_tasks.celery_tasks import (
cache_roles_across_accounts,
cache_roles_for_account,
)
from consoleme.config.config import CONFIG
old_config = copy.deepcopy(CONFIG.config)
CONFIG.config = {
**CONFIG.config,
"aws": {
**CONFIG.config.get("aws", {}),
"iamroles_redis_key":
"test_cache_roles_for_account",
},
"cache_roles_across_accounts": {
"all_roles_combined": {
"s3": {
"file":
"TestCloudCredentialAuthorizationMapping.json.gz",
}
}
},
}
cache_roles_for_account("123456789012")
cache_roles_across_accounts()
CONFIG.config = old_config
def setUpClass(cls):
from consoleme.celery_tasks.celery_tasks import (
cache_roles_across_accounts,
cache_roles_for_account,
)
cache_roles_for_account("123456789012")
cache_roles_across_accounts()
def populate_caches(
redis,
user_iam_role,
iam_sync_roles,
dummy_users_data,
dummy_requests_data,
policy_requests_table,
iamrole_table,
create_default_resources,
s3,
sns,
sqs,
iam,
www_user,
parliament,
):
from asgiref.sync import async_to_sync
from consoleme.celery_tasks import celery_tasks as celery
from consoleme.lib.account_indexers import get_account_id_to_name_mapping
from consoleme_default_plugins.plugins.celery_tasks import (
celery_tasks as default_celery_tasks, )
celery.cache_cloud_account_mapping()
accounts_d = async_to_sync(get_account_id_to_name_mapping)()
default_celery_tasks.cache_application_information()
for account_id in accounts_d.keys():
celery.cache_roles_for_account(account_id)
celery.cache_s3_buckets_for_account(account_id)
celery.cache_sns_topics_for_account(account_id)
celery.cache_sqs_queues_for_account(account_id)
celery.cache_managed_policies_for_account(account_id)
# celery.cache_resources_from_aws_config_for_account(account_id) # No select_resource_config in moto yet
# Running cache_roles_across_accounts ensures that all of the pre-existing roles in our role cache are stored in
# (mock) S3
celery.cache_roles_across_accounts()
celery.cache_policies_table_details()
celery.cache_policy_requests()
celery.cache_credential_authorization_mapping()
def create_default_resources(s3, iam, redis, iam_sync_roles, iamrole_table):
from asgiref.sync import async_to_sync
from consoleme.config import config
from consoleme.lib.cache import store_json_results_in_redis_and_s3
global all_roles
buckets = [config.get("consoleme_s3_bucket")]
for bucket in buckets:
s3.create_bucket(Bucket=bucket)
if all_roles:
async_to_sync(store_json_results_in_redis_and_s3)(
all_roles,
s3_bucket=config.get(
"cache_roles_across_accounts.all_roles_combined.s3.bucket"),
s3_key=config.get(
"cache_roles_across_accounts.all_roles_combined.s3.file"),
)
return
from consoleme.celery_tasks.celery_tasks import cache_roles_for_account
from consoleme.lib.account_indexers import get_account_id_to_name_mapping
from consoleme.lib.redis import RedisHandler
red = RedisHandler().redis_sync()
accounts_d = async_to_sync(get_account_id_to_name_mapping)()
for account_id in accounts_d.keys():
cache_roles_for_account(account_id)
cache_key = config.get("aws.iamroles_redis_key", "IAM_ROLE_CACHE")
all_roles = red.hgetall(cache_key)
async_to_sync(store_json_results_in_redis_and_s3)(
all_roles,
s3_bucket=config.get(
"cache_roles_across_accounts.all_roles_combined.s3.bucket"),
s3_key=config.get(
"cache_roles_across_accounts.all_roles_combined.s3.file"),
)
# `celery -A consoleme.celery_tasks.celery_tasks worker -l DEBUG -B -E --concurrency=8`
celery.cache_roles_across_accounts()
celery.cache_s3_buckets_across_accounts()
celery.cache_sns_topics_across_accounts()
celery.cache_sqs_queues_across_accounts()
celery.cache_managed_policies_across_accounts()
default_celery_tasks.cache_application_information()
celery.cache_resources_from_aws_config_across_accounts()
celery.cache_policies_table_details.apply_async(countdown=180)
celery.cache_policy_requests()
celery.cache_credential_authorization_mapping.apply_async(countdown=180)
else:
celery.cache_cloud_account_mapping()
accounts_d = async_to_sync(get_account_id_to_name_mapping)()
default_celery_tasks.cache_application_information()
for account_id in accounts_d.keys():
celery.cache_roles_for_account(account_id)
celery.cache_s3_buckets_for_account(account_id)
celery.cache_sns_topics_for_account(account_id)
celery.cache_sqs_queues_for_account(account_id)
celery.cache_managed_policies_for_account(account_id)
celery.cache_resources_from_aws_config_for_account(account_id)
celery.cache_policies_table_details()
celery.cache_policy_requests()
celery.cache_credential_authorization_mapping()
print("Done caching redis data")