def get_table_resource(): region = config("dynamodb_region", namespace="cis", default="us-west-2") environment = config("environment", namespace="cis", default="local") table_name = "{}-identity-vault".format(environment) if environment == "local": dynalite_host = config("dynalite_host", namespace="cis", default="localhost") dynalite_port = config("dynalite_port", namespace="cis", default="4567") session = Stubber(boto3.session.Session(region_name=region)).client resource = session.resource("dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port)) else: session = boto3.session.Session(region_name=region) resource = session.resource("dynamodb") table = resource.Table(table_name) return table
class AWS(object): """ Contains all the code necessary for role assumption in the target AWS account which holds CIS data, enumerating dynamodb-tables, and enumerating kinesis streams. """ def __init__(self): self.config = get_config() self.assume_role_session = None self._boto_session = None def session(self, region_name=None): """Return a boto_session in the current account in the current region.""" if region_name is None: # Default to us-west-2 if no region provided. logger.debug( "No region provided. Defaulting boto session to us-west-2.") region_name = "us-west-2" if self._discover_cis_environment() == "local": # If we are running the lib locally return a botocore stub. # Must call the .client object in order to return full boto session from Stubber. logger.debug( "Local environment detected. Returning boto stub session.") self._boto_session = Stubber( boto3.session.Session(region_name=region_name)).client if self._boto_session: logger.debug( "A boto session already exists on the object. Returning already constructed session." ) else: logger.debug("Initializing new boto session for region: {}".format( region_name)) self._boto_session = boto3.session.Session(region_name=region_name) return self._boto_session def assume_role(self): """Use the boto session in the current account to assume a role passed in. """ if self._discover_cis_environment() == "local": self.assume_role_session = { "Credentials": { "AccessKeyId": "FAKEAKIA", "SecretAccessKey": "FAKEACCESSKEY", "SessionToken": "FAKESESSIONTOKEN", "Expiration": (datetime.utcnow() + timedelta(hours=1)).replace(tzinfo=None), }, "AssumedRoleUser": { "AssumedRoleId": "FAKEID", "Arn": "arn:aws:iam::123456789000:role/demo-assume-role", }, "PackedPolicySize": 123, } return self.assume_role_session if self.assume_role_session is not None and self._assume_role_is_expired( ) is False: return self.assume_role_session role_arn = self.config("assume_role_arn", namespace="cis", default="None").lower() logger.debug("Role arn provided is: {}".format(role_arn)) if role_arn == "none" or role_arn == "None": logger.debug( "Assume role arn not present. Skipping assume role operation." ) res = None else: sts = self._boto_session.client("sts") res = sts.assume_role(DurationSeconds=3600, RoleArn=role_arn, RoleSessionName="cis-aws-library") self.assume_role_session = res return res def identity_vault_client(self): """Discover DynamoDb table for the environment. Return a dictionary with a client and database arn""" self.assume_role() self._check_sessions_exist() if self._discover_cis_environment() == "local": # Assume we are using dynalite and setup for that dynalite_port = self.config("dynalite_port", namespace="cis", default="4567") dynalite_host = self.config("dynalite_host", namespace="cis", default="localhost") # Initialize a dynamodb client pointed at the dynalite endpoint dynamodb_client = self._boto_session.client( "dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port)) dynamodb_resource = self._boto_session.resource( "dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port)) # Construct a dictionary of standard information. identity_vault_info = { "client": dynamodb_client, "arn": self._discover_dynamo_table(dynamodb_client), "table": dynamodb_resource.Table( self._discover_dynamo_table(dynamodb_client).split("/") [1]), } else: if self.assume_role_session is not None: # Assume we are using an assumeRole because not local. dynamodb_client = self._boto_session.client( "dynamodb", aws_access_key_id=self.assume_role_session["Credentials"] ["AccessKeyId"], aws_secret_access_key=self. assume_role_session["Credentials"]["SecretAccessKey"], aws_session_token=self.assume_role_session["Credentials"] ["SessionToken"], ) dynamodb_resource = self._boto_session.resource( "dynamodb", aws_access_key_id=self.assume_role_session["Credentials"] ["AccessKeyId"], aws_secret_access_key=self. assume_role_session["Credentials"]["SecretAccessKey"], aws_session_token=self.assume_role_session["Credentials"] ["SessionToken"], ) identity_vault_info = { "client": dynamodb_client, "arn": self._discover_dynamo_table(dynamodb_client), "table": dynamodb_resource.Table( self._discover_dynamo_table(dynamodb_client).split("/") [1]), } else: # Assume we are using in a place that uses normal credentials. dynamodb_client = self._boto_session.client("dynamodb") dynamodb_resource = self._boto_session.resource("dynamodb") identity_vault_info = { "client": dynamodb_client, "arn": self._discover_dynamo_table(dynamodb_client), "table": dynamodb_resource.Table( self._discover_dynamo_table(dynamodb_client).split("/") [1]), } return identity_vault_info def input_stream_client(self): """Discover the input stream ARN for the cis_environment. Return a dictionary containing a kinesis client and the stream arn.""" self.assume_role() self._check_sessions_exist() if self._discover_cis_environment() == "local": # Assume we are using dynalite and setup for that kinesalite_port = self.config("kinesalite_port", namespace="cis", default="4567") kinesalite_host = self.config("kinesalite_host", namespace="cis", default="localhost") # Initialize a kinesis client pointed at the kinesalite endpoint kinesis_client = self._boto_session.client( "kinesis", endpoint_url="http://{}:{}".format(kinesalite_host, kinesalite_port)) # Construct a dictionary of standard information. stream_info = { "client": kinesis_client, "arn": self._discover_kinesis_stream(kinesis_client) } else: if self.assume_role_session is not None: # Assume we are using an assumeRole because not local. kinesis_client = self._boto_session.client( "kinesis", aws_access_key_id=self.assume_role_session["Credentials"] ["AccessKeyId"], aws_secret_access_key=self. assume_role_session["Credentials"]["SecretAccessKey"], aws_session_token=self.assume_role_session["Credentials"] ["SessionToken"], ) else: # Assume we are running somwhere that can assume role natively. kinesis_client = self._boto_session.client("kinesis") stream_info = { "client": kinesis_client, "arn": self._discover_kinesis_stream(kinesis_client) } return stream_info def _check_sessions_exist(self): if self._discover_cis_environment() == "local": logger.info( "CIS Local environment detected skipping cloud based validations." ) return if self._boto_session is not None and self.assume_role_session is not None: logger.info( "Boto3 session object and assumeRole exists proceeding to next check." ) return if self._boto_session is not None: logger.info( "Running without assumeRole. Likely an ec2 instance or lambda." ) return else: logger.error("You must initialize an assumeRole and boto session.") raise ValueError( "AssumeRole or Boto3 Session not initialized. Refusing operation." ) def _assume_role_is_expired(self): if self.assume_role_session is None: return True if self._discover_cis_environment() == "local": return False expiry = self.assume_role_session["Credentials"]["Expiration"] now = datetime.utcnow() if expiry.replace(tzinfo=None) > now.replace(tzinfo=None): return False else: return True def _discover_cis_environment(self): """Use everett config manager to determine the environment we are in.""" result = self.config("environment", namespace="cis", default="local").lower() return result def _discover_kinesis_stream(self, kinesis_client): """Enumerate all kinesis streams in the region for the current assumeRole. Return the stream arn matching the appropriate tagging configuration.""" kinesis_arn = self.config("kinesis_arn", namespace="cis", default="None") if kinesis_arn != "None": return kinesis_arn if self._discover_cis_environment() == "local": # Assume developer environment and return for an explicit stream name. return kinesis_client.describe_stream( StreamName="local-stream")["StreamDescription"]["StreamARN"] else: # Assume we are in AWS and list streams, describe streams, and check tags. streams = kinesis_client.list_streams(Limit=100) for stream in streams.get("StreamNames"): tags = kinesis_client.list_tags_for_stream( StreamName=stream).get("Tags") for tag in tags: if tag.get("Key") == "cis_environment" and tag.get( "Value") == self._discover_cis_environment(): return kinesis_client.describe_stream( StreamName=stream )["StreamDescription"]["StreamARN"] else: continue return None def _discover_dynamo_table(self, dynamodb_client): """Enumerate all tables in a region for the current assumerole session. Return the arn of table matching the appropriate tagging configuration.""" dynamodb_arn = self.config("dynamodb_arn", namespace="cis", default="None") if dynamodb_arn != "None": return dynamodb_arn if self._discover_cis_environment() == "local": # Assume that the local identity vault is always called local-identity-vault return dynamodb_client.describe_table( TableName="local-identity-vault")["Table"]["TableArn"] else: # Assume that we are in AWS and list tables, describe tables, and check tags. tables = dynamodb_client.list_tables(Limit=100) for table in tables.get("TableNames"): table_arn = dynamodb_client.describe_table( TableName=table)["Table"]["TableArn"] tags = dynamodb_client.list_tags_of_resource( ResourceArn=table_arn).get("Tags", []) for tag in tags: if tag.get("Key") == "cis_environment" and tag.get( "Value") == self._discover_cis_environment(): return table_arn else: continue return None
class IdentityVault(object): def __init__(self): self.boto_session = None self.dynamodb_client = None self.config = get_config() def connect(self): self._session() if self.dynamodb_client is None: if self._get_cis_environment() == "local": dynalite_port = self.config("dynalite_port", namespace="cis", default="4567") dynalite_host = self.config("dynalite_host", namespace="cis", default="localhost") self.dynamodb_client = self.boto_session.client( "dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port)) else: self.dynamodb_client = self.boto_session.client("dynamodb") return self.dynamodb_client def _session(self): if self.boto_session is None: region = self.config("region_name", namespace="cis", default="us-west-2") if self._get_cis_environment() == "local": self.boto_session = Stubber( boto3.session.Session(region_name=region)).client else: self.boto_session = boto3.session.Session(region_name=region) return self.boto_session def _get_cis_environment(self): return self.config("environment", namespace="cis", default="local") def _generate_table_name(self): return "{}-identity-vault".format(self._get_cis_environment()) def enable_stream(self): self.connect() result = self.dynamodb_client.update_table( TableName=self._generate_table_name(), StreamSpecification={ "StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES" }, ) return result def enable_autoscaler(self): scaler_config = autoscale.ScalableTable(self._generate_table_name()) scaler_config.connect() return scaler_config.enable_autoscaler() def tag_vault(self): self.connect() arn = self.find() tags = [ { "Key": "cis_environment", "Value": self._get_cis_environment() }, { "Key": "application", "Value": "identity-vault" }, ] try: return self.dynamodb_client.tag_resource(ResourceArn=arn, Tags=tags) except ClientError: logger.error("The table does not support tagging.") except Exception as e: logger.error( "The table did not tag for an unknown reason: {}".format(e)) def find(self): self.connect() try: if self._get_cis_environment() == "local": # Assume that the local identity vault is always called local-identity-vault return self.dynamodb_client.describe_table( TableName="local-identity-vault")["Table"]["TableArn"] else: # Assume that we are in AWS and list tables, describe tables, and check tags. tables = self.dynamodb_client.list_tables(Limit=100) for table in tables.get("TableNames"): table_arn = self.dynamodb_client.describe_table( TableName=table)["Table"]["TableArn"] if table == self._generate_table_name(): return table_arn except ClientError as exception: if exception.response["Error"][ "Code"] == "ResourceNotFoundException": return None else: raise def create(self): if self._get_cis_environment() not in [ "production", "development", "testing" ]: result = self.dynamodb_client.create_table( TableName=self._generate_table_name(), KeySchema=[{ "AttributeName": "id", "KeyType": "HASH" }], AttributeDefinitions=[ # auth0 user_id { "AttributeName": "id", "AttributeType": "S" }, # user_uuid formerly dinopark id (uuid is a reserverd keyword in dynamo, hence user_uuid) { "AttributeName": "user_uuid", "AttributeType": "S" }, # sequence number for the last integration { "AttributeName": "sequence_number", "AttributeType": "S" }, # value of the primary_email attribute { "AttributeName": "primary_email", "AttributeType": "S" }, # value of the primary_username attribute { "AttributeName": "primary_username", "AttributeType": "S" }, ], ProvisionedThroughput={ "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, GlobalSecondaryIndexes=[ { "IndexName": "{}-sequence_number".format( self._generate_table_name()), "KeySchema": [{ "AttributeName": "sequence_number", "KeyType": "HASH" }], "Projection": { "ProjectionType": "ALL" }, "ProvisionedThroughput": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, }, { "IndexName": "{}-primary_username".format( self._generate_table_name()), "KeySchema": [ { "AttributeName": "primary_username", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, "ProvisionedThroughput": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, }, { "IndexName": "{}-primary_email".format(self._generate_table_name()), "KeySchema": [ { "AttributeName": "primary_email", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, "ProvisionedThroughput": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, }, { "IndexName": "{}-user_uuid".format(self._generate_table_name()), "KeySchema": [ { "AttributeName": "user_uuid", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, # Removed due to moving to pay per query. "ProvisionedThroughput": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, }, ], ) else: result = self.dynamodb_client.create_table( TableName=self._generate_table_name(), KeySchema=[{ "AttributeName": "id", "KeyType": "HASH" }], AttributeDefinitions=[ # auth0 user_id { "AttributeName": "id", "AttributeType": "S" }, # user_uuid formerly dinopark id (uuid is a reserverd keyword in dynamo, hence user_uuid) { "AttributeName": "user_uuid", "AttributeType": "S" }, # sequence number for the last integration { "AttributeName": "sequence_number", "AttributeType": "S" }, # value of the primary_email attribute { "AttributeName": "primary_email", "AttributeType": "S" }, # value of the primary_username attribute { "AttributeName": "primary_username", "AttributeType": "S" }, ], BillingMode="PAY_PER_REQUEST", GlobalSecondaryIndexes=[ { "IndexName": "{}-sequence_number".format( self._generate_table_name()), "KeySchema": [{ "AttributeName": "sequence_number", "KeyType": "HASH" }], "Projection": { "ProjectionType": "ALL" }, }, { "IndexName": "{}-primary_username".format( self._generate_table_name()), "KeySchema": [ { "AttributeName": "primary_username", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, }, { "IndexName": "{}-primary_email".format(self._generate_table_name()), "KeySchema": [ { "AttributeName": "primary_email", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, }, { "IndexName": "{}-user_uuid".format(self._generate_table_name()), "KeySchema": [ { "AttributeName": "user_uuid", "KeyType": "HASH" }, { "AttributeName": "id", "KeyType": "RANGE" }, ], "Projection": { "ProjectionType": "ALL" }, }, ], ) waiter = self.dynamodb_client.get_waiter("table_exists") if self._get_cis_environment() in [ "production", "development", "testing" ]: waiter.wait(TableName=self._generate_table_name(), WaiterConfig={ "Delay": 20, "MaxAttempts": 20 }) self.tag_vault() self.setup_stream() else: waiter.wait(TableName=self._generate_table_name(), WaiterConfig={ "Delay": 1, "MaxAttempts": 5 }) return result def destroy(self): result = self.dynamodb_client.delete_table( TableName=self._generate_table_name()) return result def __get_table_resource(self): region = self.config("region_name", namespace="cis", default="us-west-2") if self._get_cis_environment() == "local": self.boto_session = Stubber( boto3.session.Session(region_name=region)).client dynalite_port = self.config("dynalite_port", namespace="cis", default="4567") dynalite_host = self.config("dynalite_host", namespace="cis", default="localhost") dynamodb_resource = self.boto_session.resource( "dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port)) table = dynamodb_resource.Table(self._generate_table_name()) else: dynamodb_resource = boto3.resource("dynamodb", region_name=region) table = dynamodb_resource.Table(self._generate_table_name()) return table def find_or_create(self): if self.find() is not None: table = self.__get_table_resource() else: self.create() table = self.__get_table_resource() return table def describe_indices(self): return self.dynamodb_client.describe_table( TableName=self._generate_table_name()) def _has_stream(self): result = self.dynamodb_client.describe_table( TableName=self._generate_table_name()).get("Table") if result.get("StreamSpecification"): return True else: return False def setup_stream(self): if self._has_stream() is False: try: return self.dynamodb_client.update_table( TableName=self._generate_table_name(), StreamSpecification={ "StreamEnabled": True, "StreamViewType": "KEYS_ONLY" }, ) except ClientError as e: logger.error( "The table does not support streams: {}.".format(e)) return except Exception as e: logger.error( "The table did not tag for an unknown reason: {}".format( e))