result.append((role, attached_name))
# Eliminate dupes
result = sorted(list(set(result)))
if do_headers:
# Add headers
result = [("Role", "Policy")] + result
return result
# ---
# dss-ops.py command line utility
# ---
iam = dispatch.target("iam", arguments={}, help=__doc__)
@iam.action(
"list",
arguments={
"cloud_provider":
dict(choices=["aws", "gcp", "fusillade"],
help="The cloud provider whose policies are being listed"),
"--group-by":
dict(
required=False,
choices=["users", "groups", "roles"],
help=
"Group the listed policies by asset type (user, group, or role)",
),
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock
from string import hexdigits
import boto3
from elasticsearch_dsl import Search
from dcplib.aws.sqs import SQSMessenger, get_queue_url
from dss import Config, Replica
from dss.operations import dispatch
from dss.api.subscriptions_v1 import _delete_subscription
from dss.index.es import ElasticsearchClient
logger = logging.getLogger(__name__)
elasticsearch = dispatch.target("elasticsearch", help=__doc__)
@elasticsearch.action("index-keys",
arguments={
"--replica":
dict(choices=[r.name for r in Replica],
required=True),
"--keys":
dict(default=None, nargs="*", help="keys to index.")
})
def index_keys(argv: typing.List[str], args: argparse.Namespace):
"""
Queue an SQS message to the indexer lambda for each key in `keys`.
"""
index_queue_url = get_queue_url("dss-index-operation-" +
from dcplib.aws.sqs import SQSMessenger
from dcplib.aws.clients import logs
from dss.config import Config, Replica
from dss.util.aws import resources
from dss.operations import dispatch
from dss.events import (get_bundle_metadata_document, record_event_for_bundle, journal_flashflood,
update_flashflood, list_new_flashflood_journals)
from dss.storage.bundles import Living
from dss.operations.util import monitor_logs, command_queue_url
logger = logging.getLogger(__name__)
events = dispatch.target("events", help=__doc__)
@events.action("bundle-metadata-document",
arguments={"--replica": dict(choices=[r.name for r in Replica], required=True),
"--keys": dict(required=True,
nargs="*",
help="bundle keys to generate documents.")})
def bundle_metadata_document(argv: typing.List[str], args: argparse.Namespace):
replica = Replica[args.replica]
for key in args.keys:
md = get_bundle_metadata_document(replica, key)
if md is not None:
print(md)
@events.action("record",
try:
response = sm_client.get_secret_value(SecretId=secret_name)
except ClientError as e:
if "Error" in e.response:
errtype = e.response["Error"]["Code"]
if errtype == "ResourceNotFoundException":
raise RuntimeError(
f"Error: secret {secret_name} was not found!")
raise RuntimeError(
f"Error: could not fetch secret {secret_name} from secrets manager"
)
else:
return response
secrets = dispatch.target("secrets", arguments={}, help=__doc__)
json_flag_options = dict(
default=False,
action="store_true",
help="format the output as JSON if this flag is present")
dryrun_flag_options = dict(default=False,
action="store_true",
help="do a dry run of the actual operation")
quiet_flag_options = dict(default=False,
action="store_true",
help="suppress output")
@secrets.action(
"list",
import json
import typing
import logging
import argparse
from collections import namedtuple
from cloud_blobstore import BlobNotFoundError
from dss import Config, Replica
from dcplib.aws.sqs import SQSMessenger, get_queue_url
from dss.storage.hcablobstore import compose_blob_key
from dss.operations import dispatch
from dss.storage.identifiers import BLOB_PREFIX, FILE_PREFIX, BUNDLE_PREFIX
logger = logging.getLogger(__name__)
sync = dispatch.target("sync", help=__doc__)
ReplicationAnomaly = namedtuple("ReplicationAnomaly", "key anomaly")
def _log_warning(**kwargs):
logger.warning(json.dumps(kwargs))
@sync.action("verify-sync",
arguments={
"--source-replica":
dict(choices=[r.name for r in Replica], required=True),
"--destination-replica":
dict(choices=[r.name for r in Replica], required=True),
"--keys":
dict(default=None, nargs="*", help="keys to check.")
def _parse_key(key):
try:
version = VERSION_REGEX.search(key).group(0)
uuid = UUID_REGEX.search(key).group(0)
except IndexError:
raise RuntimeError(f'Unable to parse the key: {key}')
return uuid, version
def process_keys(self):
raise NotImplementedError()
def __call__(self, argv: typing.List[str], args: argparse.Namespace):
self.process_keys()
checkout = dispatch.target("checkout", help=__doc__)
@checkout.action("remove",
arguments={
"--replica":
dict(choices=[r.name for r in Replica], required=True),
"--keys":
dict(nargs="+",
help="Keys to remove from checkout.",
required=True)
})
class Remove(CheckoutHandler):
def process_keys(self):
"""Remove keys from the checkout bucket."""
for _key in self.keys:
quiet,
f"Nothing to unset for variable {env_var} in deployed lambda function {lambda_name}"
)
def print_lambda_env(lambda_name, lambda_env):
"""Print environment variables set in a specified lambda function"""
print(f"\n{lambda_name}:")
for name, val in lambda_env.items():
print(f"{name}={val}")
# ---
# Command line utility functions
# ---
lambda_params = dispatch.target("lambda", arguments={}, help=__doc__)
ssm_params = dispatch.target("params", arguments={}, help=__doc__)
json_flag_options = dict(
default=False,
action="store_true",
help="format the output as JSON if this flag is present")
dryrun_flag_options = dict(default=False,
action="store_true",
help="do a dry run of the actual operation")
quiet_flag_options = dict(default=False,
action="store_true",
help="suppress output")
@ssm_params.action(