def get_table_resource():
region = config("dynamodb_region", namespace="cis", default="us-west-2")
environment = config("environment", namespace="cis", default="local")
table_name = "{}-identity-vault".format(environment)
if environment == "local":
dynalite_host = config("dynalite_host", namespace="cis", default="localhost")
dynalite_port = config("dynalite_port", namespace="cis", default="4567")
session = Stubber(boto3.session.Session(region_name=region)).client
resource = session.resource("dynamodb", endpoint_url="http://{}:{}".format(dynalite_host, dynalite_port))
else:
session = boto3.session.Session(region_name=region)
resource = session.resource("dynamodb")
table = resource.Table(table_name)
return table
class AWS(object):
"""
Contains all the code necessary for role assumption in
the target AWS account which holds CIS data,
enumerating dynamodb-tables, and enumerating kinesis streams.
"""
def __init__(self):
self.config = get_config()
self.assume_role_session = None
self._boto_session = None
def session(self, region_name=None):
"""Return a boto_session in the current account
in the current region."""
if region_name is None:
# Default to us-west-2 if no region provided.
logger.debug(
"No region provided. Defaulting boto session to us-west-2.")
region_name = "us-west-2"
if self._discover_cis_environment() == "local":
# If we are running the lib locally return a botocore stub.
# Must call the .client object in order to return full boto session from Stubber.
logger.debug(
"Local environment detected. Returning boto stub session.")
self._boto_session = Stubber(
boto3.session.Session(region_name=region_name)).client
if self._boto_session:
logger.debug(
"A boto session already exists on the object. Returning already constructed session."
)
else:
logger.debug("Initializing new boto session for region: {}".format(
region_name))
self._boto_session = boto3.session.Session(region_name=region_name)
return self._boto_session
def assume_role(self):
"""Use the boto session in the current account
to assume a role passed in.
"""
if self._discover_cis_environment() == "local":
self.assume_role_session = {
"Credentials": {
"AccessKeyId":
"FAKEAKIA",
"SecretAccessKey":
"FAKEACCESSKEY",
"SessionToken":
"FAKESESSIONTOKEN",
"Expiration": (datetime.utcnow() +
timedelta(hours=1)).replace(tzinfo=None),
},
"AssumedRoleUser": {
"AssumedRoleId": "FAKEID",
"Arn": "arn:aws:iam::123456789000:role/demo-assume-role",
},
"PackedPolicySize": 123,
}
return self.assume_role_session
if self.assume_role_session is not None and self._assume_role_is_expired(
) is False:
return self.assume_role_session
role_arn = self.config("assume_role_arn",
namespace="cis",
default="None").lower()
logger.debug("Role arn provided is: {}".format(role_arn))
if role_arn == "none" or role_arn == "None":
logger.debug(
"Assume role arn not present. Skipping assume role operation."
)
res = None
else:
sts = self._boto_session.client("sts")
res = sts.assume_role(DurationSeconds=3600,
RoleArn=role_arn,
RoleSessionName="cis-aws-library")
self.assume_role_session = res
return res
def identity_vault_client(self):
"""Discover DynamoDb table for the environment.
Return a dictionary with a client and database arn"""
self.assume_role()
self._check_sessions_exist()
if self._discover_cis_environment() == "local":
# Assume we are using dynalite and setup for that
dynalite_port = self.config("dynalite_port",
namespace="cis",
default="4567")
dynalite_host = self.config("dynalite_host",
namespace="cis",
default="localhost")
# Initialize a dynamodb client pointed at the dynalite endpoint
dynamodb_client = self._boto_session.client(
"dynamodb",
endpoint_url="http://{}:{}".format(dynalite_host,
dynalite_port))
dynamodb_resource = self._boto_session.resource(
"dynamodb",
endpoint_url="http://{}:{}".format(dynalite_host,
dynalite_port))
# Construct a dictionary of standard information.
identity_vault_info = {
"client":
dynamodb_client,
"arn":
self._discover_dynamo_table(dynamodb_client),
"table":
dynamodb_resource.Table(
self._discover_dynamo_table(dynamodb_client).split("/")
[1]),
}
else:
if self.assume_role_session is not None:
# Assume we are using an assumeRole because not local.
dynamodb_client = self._boto_session.client(
"dynamodb",
aws_access_key_id=self.assume_role_session["Credentials"]
["AccessKeyId"],
aws_secret_access_key=self.
assume_role_session["Credentials"]["SecretAccessKey"],
aws_session_token=self.assume_role_session["Credentials"]
["SessionToken"],
)
dynamodb_resource = self._boto_session.resource(
"dynamodb",
aws_access_key_id=self.assume_role_session["Credentials"]
["AccessKeyId"],
aws_secret_access_key=self.
assume_role_session["Credentials"]["SecretAccessKey"],
aws_session_token=self.assume_role_session["Credentials"]
["SessionToken"],
)
identity_vault_info = {
"client":
dynamodb_client,
"arn":
self._discover_dynamo_table(dynamodb_client),
"table":
dynamodb_resource.Table(
self._discover_dynamo_table(dynamodb_client).split("/")
[1]),
}
else:
# Assume we are using in a place that uses normal credentials.
dynamodb_client = self._boto_session.client("dynamodb")
dynamodb_resource = self._boto_session.resource("dynamodb")
identity_vault_info = {
"client":
dynamodb_client,
"arn":
self._discover_dynamo_table(dynamodb_client),
"table":
dynamodb_resource.Table(
self._discover_dynamo_table(dynamodb_client).split("/")
[1]),
}
return identity_vault_info
def input_stream_client(self):
"""Discover the input stream ARN for the cis_environment.
Return a dictionary containing a kinesis client and the stream arn."""
self.assume_role()
self._check_sessions_exist()
if self._discover_cis_environment() == "local":
# Assume we are using dynalite and setup for that
kinesalite_port = self.config("kinesalite_port",
namespace="cis",
default="4567")
kinesalite_host = self.config("kinesalite_host",
namespace="cis",
default="localhost")
# Initialize a kinesis client pointed at the kinesalite endpoint
kinesis_client = self._boto_session.client(
"kinesis",
endpoint_url="http://{}:{}".format(kinesalite_host,
kinesalite_port))
# Construct a dictionary of standard information.
stream_info = {
"client": kinesis_client,
"arn": self._discover_kinesis_stream(kinesis_client)
}
else:
if self.assume_role_session is not None:
# Assume we are using an assumeRole because not local.
kinesis_client = self._boto_session.client(
"kinesis",
aws_access_key_id=self.assume_role_session["Credentials"]
["AccessKeyId"],
aws_secret_access_key=self.
assume_role_session["Credentials"]["SecretAccessKey"],
aws_session_token=self.assume_role_session["Credentials"]
["SessionToken"],
)
else:
# Assume we are running somwhere that can assume role natively.
kinesis_client = self._boto_session.client("kinesis")
stream_info = {
"client": kinesis_client,
"arn": self._discover_kinesis_stream(kinesis_client)
}
return stream_info
def _check_sessions_exist(self):
if self._discover_cis_environment() == "local":
logger.info(
"CIS Local environment detected skipping cloud based validations."
)
return
if self._boto_session is not None and self.assume_role_session is not None:
logger.info(
"Boto3 session object and assumeRole exists proceeding to next check."
)
return
if self._boto_session is not None:
logger.info(
"Running without assumeRole. Likely an ec2 instance or lambda."
)
return
else:
logger.error("You must initialize an assumeRole and boto session.")
raise ValueError(
"AssumeRole or Boto3 Session not initialized. Refusing operation."
)
def _assume_role_is_expired(self):
if self.assume_role_session is None:
return True
if self._discover_cis_environment() == "local":
return False
expiry = self.assume_role_session["Credentials"]["Expiration"]
now = datetime.utcnow()
if expiry.replace(tzinfo=None) > now.replace(tzinfo=None):
return False
else:
return True
def _discover_cis_environment(self):
"""Use everett config manager to determine the environment we are in."""
result = self.config("environment", namespace="cis",
default="local").lower()
return result
def _discover_kinesis_stream(self, kinesis_client):
"""Enumerate all kinesis streams in the region for the current
assumeRole. Return the stream arn matching the appropriate tagging
configuration."""
kinesis_arn = self.config("kinesis_arn",
namespace="cis",
default="None")
if kinesis_arn != "None":
return kinesis_arn
if self._discover_cis_environment() == "local":
# Assume developer environment and return for an explicit stream name.
return kinesis_client.describe_stream(
StreamName="local-stream")["StreamDescription"]["StreamARN"]
else:
# Assume we are in AWS and list streams, describe streams, and check tags.
streams = kinesis_client.list_streams(Limit=100)
for stream in streams.get("StreamNames"):
tags = kinesis_client.list_tags_for_stream(
StreamName=stream).get("Tags")
for tag in tags:
if tag.get("Key") == "cis_environment" and tag.get(
"Value") == self._discover_cis_environment():
return kinesis_client.describe_stream(
StreamName=stream
)["StreamDescription"]["StreamARN"]
else:
continue
return None
def _discover_dynamo_table(self, dynamodb_client):
"""Enumerate all tables in a region for the current
assumerole session. Return the arn of table matching the
appropriate tagging configuration."""
dynamodb_arn = self.config("dynamodb_arn",
namespace="cis",
default="None")
if dynamodb_arn != "None":
return dynamodb_arn
if self._discover_cis_environment() == "local":
# Assume that the local identity vault is always called local-identity-vault
return dynamodb_client.describe_table(
TableName="local-identity-vault")["Table"]["TableArn"]
else:
# Assume that we are in AWS and list tables, describe tables, and check tags.
tables = dynamodb_client.list_tables(Limit=100)
for table in tables.get("TableNames"):
table_arn = dynamodb_client.describe_table(
TableName=table)["Table"]["TableArn"]
tags = dynamodb_client.list_tags_of_resource(
ResourceArn=table_arn).get("Tags", [])
for tag in tags:
if tag.get("Key") == "cis_environment" and tag.get(
"Value") == self._discover_cis_environment():
return table_arn
else:
continue
return None
class IdentityVault(object):
def __init__(self):
self.boto_session = None
self.dynamodb_client = None
self.config = get_config()
def connect(self):
self._session()
if self.dynamodb_client is None:
if self._get_cis_environment() == "local":
dynalite_port = self.config("dynalite_port",
namespace="cis",
default="4567")
dynalite_host = self.config("dynalite_host",
namespace="cis",
default="localhost")
self.dynamodb_client = self.boto_session.client(
"dynamodb",
endpoint_url="http://{}:{}".format(dynalite_host,
dynalite_port))
else:
self.dynamodb_client = self.boto_session.client("dynamodb")
return self.dynamodb_client
def _session(self):
if self.boto_session is None:
region = self.config("region_name",
namespace="cis",
default="us-west-2")
if self._get_cis_environment() == "local":
self.boto_session = Stubber(
boto3.session.Session(region_name=region)).client
else:
self.boto_session = boto3.session.Session(region_name=region)
return self.boto_session
def _get_cis_environment(self):
return self.config("environment", namespace="cis", default="local")
def _generate_table_name(self):
return "{}-identity-vault".format(self._get_cis_environment())
def enable_stream(self):
self.connect()
result = self.dynamodb_client.update_table(
TableName=self._generate_table_name(),
StreamSpecification={
"StreamEnabled": True,
"StreamViewType": "NEW_AND_OLD_IMAGES"
},
)
return result
def enable_autoscaler(self):
scaler_config = autoscale.ScalableTable(self._generate_table_name())
scaler_config.connect()
return scaler_config.enable_autoscaler()
def tag_vault(self):
self.connect()
arn = self.find()
tags = [
{
"Key": "cis_environment",
"Value": self._get_cis_environment()
},
{
"Key": "application",
"Value": "identity-vault"
},
]
try:
return self.dynamodb_client.tag_resource(ResourceArn=arn,
Tags=tags)
except ClientError:
logger.error("The table does not support tagging.")
except Exception as e:
logger.error(
"The table did not tag for an unknown reason: {}".format(e))
def find(self):
self.connect()
try:
if self._get_cis_environment() == "local":
# Assume that the local identity vault is always called local-identity-vault
return self.dynamodb_client.describe_table(
TableName="local-identity-vault")["Table"]["TableArn"]
else:
# Assume that we are in AWS and list tables, describe tables, and check tags.
tables = self.dynamodb_client.list_tables(Limit=100)
for table in tables.get("TableNames"):
table_arn = self.dynamodb_client.describe_table(
TableName=table)["Table"]["TableArn"]
if table == self._generate_table_name():
return table_arn
except ClientError as exception:
if exception.response["Error"][
"Code"] == "ResourceNotFoundException":
return None
else:
raise
def create(self):
if self._get_cis_environment() not in [
"production", "development", "testing"
]:
result = self.dynamodb_client.create_table(
TableName=self._generate_table_name(),
KeySchema=[{
"AttributeName": "id",
"KeyType": "HASH"
}],
AttributeDefinitions=[
# auth0 user_id
{
"AttributeName": "id",
"AttributeType": "S"
},
# user_uuid formerly dinopark id (uuid is a reserverd keyword in dynamo, hence user_uuid)
{
"AttributeName": "user_uuid",
"AttributeType": "S"
},
# sequence number for the last integration
{
"AttributeName": "sequence_number",
"AttributeType": "S"
},
# value of the primary_email attribute
{
"AttributeName": "primary_email",
"AttributeType": "S"
},
# value of the primary_username attribute
{
"AttributeName": "primary_username",
"AttributeType": "S"
},
],
ProvisionedThroughput={
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
GlobalSecondaryIndexes=[
{
"IndexName":
"{}-sequence_number".format(
self._generate_table_name()),
"KeySchema": [{
"AttributeName": "sequence_number",
"KeyType": "HASH"
}],
"Projection": {
"ProjectionType": "ALL"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
},
{
"IndexName":
"{}-primary_username".format(
self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "primary_username",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
},
{
"IndexName":
"{}-primary_email".format(self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "primary_email",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
},
{
"IndexName":
"{}-user_uuid".format(self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "user_uuid",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
# Removed due to moving to pay per query.
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
},
],
)
else:
result = self.dynamodb_client.create_table(
TableName=self._generate_table_name(),
KeySchema=[{
"AttributeName": "id",
"KeyType": "HASH"
}],
AttributeDefinitions=[
# auth0 user_id
{
"AttributeName": "id",
"AttributeType": "S"
},
# user_uuid formerly dinopark id (uuid is a reserverd keyword in dynamo, hence user_uuid)
{
"AttributeName": "user_uuid",
"AttributeType": "S"
},
# sequence number for the last integration
{
"AttributeName": "sequence_number",
"AttributeType": "S"
},
# value of the primary_email attribute
{
"AttributeName": "primary_email",
"AttributeType": "S"
},
# value of the primary_username attribute
{
"AttributeName": "primary_username",
"AttributeType": "S"
},
],
BillingMode="PAY_PER_REQUEST",
GlobalSecondaryIndexes=[
{
"IndexName":
"{}-sequence_number".format(
self._generate_table_name()),
"KeySchema": [{
"AttributeName": "sequence_number",
"KeyType": "HASH"
}],
"Projection": {
"ProjectionType": "ALL"
},
},
{
"IndexName":
"{}-primary_username".format(
self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "primary_username",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
},
{
"IndexName":
"{}-primary_email".format(self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "primary_email",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
},
{
"IndexName":
"{}-user_uuid".format(self._generate_table_name()),
"KeySchema": [
{
"AttributeName": "user_uuid",
"KeyType": "HASH"
},
{
"AttributeName": "id",
"KeyType": "RANGE"
},
],
"Projection": {
"ProjectionType": "ALL"
},
},
],
)
waiter = self.dynamodb_client.get_waiter("table_exists")
if self._get_cis_environment() in [
"production", "development", "testing"
]:
waiter.wait(TableName=self._generate_table_name(),
WaiterConfig={
"Delay": 20,
"MaxAttempts": 20
})
self.tag_vault()
self.setup_stream()
else:
waiter.wait(TableName=self._generate_table_name(),
WaiterConfig={
"Delay": 1,
"MaxAttempts": 5
})
return result
def destroy(self):
result = self.dynamodb_client.delete_table(
TableName=self._generate_table_name())
return result
def __get_table_resource(self):
region = self.config("region_name",
namespace="cis",
default="us-west-2")
if self._get_cis_environment() == "local":
self.boto_session = Stubber(
boto3.session.Session(region_name=region)).client
dynalite_port = self.config("dynalite_port",
namespace="cis",
default="4567")
dynalite_host = self.config("dynalite_host",
namespace="cis",
default="localhost")
dynamodb_resource = self.boto_session.resource(
"dynamodb",
endpoint_url="http://{}:{}".format(dynalite_host,
dynalite_port))
table = dynamodb_resource.Table(self._generate_table_name())
else:
dynamodb_resource = boto3.resource("dynamodb", region_name=region)
table = dynamodb_resource.Table(self._generate_table_name())
return table
def find_or_create(self):
if self.find() is not None:
table = self.__get_table_resource()
else:
self.create()
table = self.__get_table_resource()
return table
def describe_indices(self):
return self.dynamodb_client.describe_table(
TableName=self._generate_table_name())
def _has_stream(self):
result = self.dynamodb_client.describe_table(
TableName=self._generate_table_name()).get("Table")
if result.get("StreamSpecification"):
return True
else:
return False
def setup_stream(self):
if self._has_stream() is False:
try:
return self.dynamodb_client.update_table(
TableName=self._generate_table_name(),
StreamSpecification={
"StreamEnabled": True,
"StreamViewType": "KEYS_ONLY"
},
)
except ClientError as e:
logger.error(
"The table does not support streams: {}.".format(e))
return
except Exception as e:
logger.error(
"The table did not tag for an unknown reason: {}".format(
e))
