def test_datastore():
""" Tries to save garbage data to the DB """
datastore = Datastore()
mydict = {"fingerprint": "9d:bc:c5:f3:a6:12:9e:0b:b5:f3:3c:93:0e:32:78:80:9c:a9:ce:8c"}
datastore.store("keypair", "us-east-1", "seg", "myname", True, mydict)
for itemrevision in datastore.get("keypair", "us-east-1", "seg", "myname"):
print itemrevision.__dict__
def ensure_item_has_latest_revision_id(item):
"""
This is a function that will attempt to correct an item that does not have a latest revision id set.
There are two outcomes that result:
1. If there is a revision with the item id, find the latest revision, and update the item such that it
point to that latest revision.
2. If not -- then we will treat the item as rancid and delete it from the database.
:param item:
:return The item if it was fixed -- or None if it was deleted from the DB:
"""
if not item.latest_revision_id:
current_revision = db.session.query(ItemRevision).filter(ItemRevision.item_id == item.id)\
.order_by(ItemRevision.date_created.desc()).first()
if not current_revision:
db.session.delete(item)
db.session.commit()
app.logger.error("[?] Item: {name}/{tech}/{account}/{region} does NOT have a latest revision attached, "
"and no current revisions were located. The item has been deleted.".format(
name=item.name,
tech=item.technology.name,
account=item.account.name,
region=item.region))
return None
else:
# Update the latest revision ID:
item.latest_revision_id = current_revision.id
# Also need to generate the hashes:
# 1. Get the watcher class of the item:
watcher_cls = watcher_registry[item.technology.name]
watcher = watcher_cls(accounts=[item.account.name])
ds = Datastore()
# 2. Generate the hashes:
if watcher.honor_ephemerals:
ephemeral_paths = watcher.ephemeral_paths
else:
ephemeral_paths = []
item.latest_revision_complete_hash = ds.hash_config(current_revision.config)
item.latest_revision_durable_hash = ds.durable_hash(current_revision.config, ephemeral_paths)
db.session.add(item)
db.session.commit()
app.logger.error("[?] Item: {name}/{tech}/{account}/{region} does NOT have a latest revision attached, "
"but a current revision was located. The item has been fixed.".format(
name=item.name,
tech=item.technology.name,
account=item.account.name,
region=item.region))
return item
def ensure_item_has_latest_revision_id(item):
"""
This is a function that will attempt to correct an item that does not have a latest revision id set.
There are two outcomes that result:
1. If there is a revision with the item id, find the latest revision, and update the item such that it
point to that latest revision.
2. If not -- then we will treat the item as rancid and delete it from the database.
:param item:
:return The item if it was fixed -- or None if it was deleted from the DB:
"""
if not item.latest_revision_id:
current_revision = db.session.query(ItemRevision).filter(ItemRevision.item_id == item.id)\
.order_by(ItemRevision.date_created.desc()).first()
if not current_revision:
db.session.delete(item)
db.session.commit()
app.logger.error("[?] Item: {name}/{tech}/{account}/{region} does NOT have a latest revision attached, "
"and no current revisions were located. The item has been deleted.".format(
name=item.name,
tech=item.technology.name,
account=item.account.name,
region=item.region))
return None
else:
# Update the latest revision ID:
item.latest_revision_id = current_revision.id
# Also need to generate the hashes:
# 1. Get the watcher class of the item:
watcher_cls = watcher_registry[item.technology.name]
watcher = watcher_cls(accounts=[item.account.name])
ds = Datastore()
# 2. Generate the hashes:
if watcher.honor_ephemerals:
ephemeral_paths = watcher.ephemeral_paths
else:
ephemeral_paths = []
item.latest_revision_complete_hash = ds.hash_config(current_revision.config)
item.latest_revision_durable_hash = ds.durable_hash(current_revision.config, ephemeral_paths)
db.session.add(item)
db.session.commit()
app.logger.error("[?] Item: {name}/{tech}/{account}/{region} does NOT have a latest revision attached, "
"but a current revision was located. The item has been fixed.".format(
name=item.name,
tech=item.technology.name,
account=item.account.name,
region=item.region))
return item
def test_datastore():
""" Tries to save garbage data to the DB """
datastore = Datastore()
mydict = {
"fingerprint":
"9d:bc:c5:f3:a6:12:9e:0b:b5:f3:3c:93:0e:32:78:80:9c:a9:ce:8c"
}
datastore.store("keypair", "us-east-1", "seg", "myname", True, mydict)
for itemrevision in datastore.get("keypair", "us-east-1", "seg", "myname"):
print itemrevision.__dict__
def test_save_ephemeral_changed_item(self):
self._setup_account()
datastore = Datastore()
old_item = ChangeItem(index='test_index',
account='test_account',
name='item_name',
active=True,
new_config={'config': 'test1'})
old_item.save(datastore)
query = Item.query.filter(Technology.name == 'test_index').filter(
Account.name == 'test_account')
items = query.all()
self.assertEqual(len(items), 1)
revisions = items[0].revisions.all()
self.assertEqual(len(revisions), 1)
new_item = ChangeItem(index='test_index',
account='test_account',
name='item_name',
active=True,
new_config={'config': 'test2'})
watcher = Watcher(accounts=['test_account'])
watcher.index = 'test_index'
watcher.honor_ephemerals = True
watcher.ephemeral_paths = ["config"]
watcher.find_changes(current=[new_item])
watcher.save()
query = Item.query.filter(Technology.name == 'test_index').filter(
Account.name == 'test_account')
items = query.all()
self.assertEqual(len(items), 1)
revisions = items[0].revisions.all()
self.assertEqual(len(revisions), 1)
def _delete_issues(settings):
account = Account.query.filter(Account.id == settings.account_id).first()
tech = Technology.query.filter(Technology.id == settings.tech_id).first()
if account and tech:
# Report issues as fixed
db_items = Datastore().get_all_ctype_filtered(tech=tech.name, account=account.name, include_inactive=False)
items = []
for item in db_items:
new_item = ChangeItem(index=tech.name,
region=item.region,
account=account.name,
name=item.name,
arn=item.arn)
new_item.audit_issues = []
new_item.db_item = item
items.append(new_item)
for item in items:
for issue in item.db_item.issues:
if issue.auditor_setting_id == settings.id:
item.confirmed_fixed_issues.append(issue)
db.session.delete(settings)
def test_ensure_item_has_latest_revision_id(self):
"""
Test that items always have a proper current revision set. Otherwise, the item needs to be deleted.
:return:
"""
from security_monkey.watchers.iam.iam_role import IAMRole
from security_monkey.watcher import ensure_item_has_latest_revision_id
from security_monkey.datastore import Datastore
# Stop the watcher registry from stepping on everyone's toes:
import security_monkey.watcher
old_watcher_registry = security_monkey.watcher.watcher_registry
security_monkey.watcher.watcher_registry = {IAMRole.index: IAMRole}
# Set everything up:
self.setup_batch_db()
watcher = IAMRole(accounts=[self.account.name])
watcher.current_account = (self.account, 0)
watcher.technology = self.technology
# Test case #1: Create an item in the DB that has no current revision ID:
no_revision_item = Item(region="us-east-1",
name="NOREVISION",
account_id=self.account.id,
tech_id=self.technology.id)
db.session.add(no_revision_item)
db.session.commit()
assert db.session.query(Item).filter(
Item.name == no_revision_item.name).one()
# Should delete the item from the DB:
result = ensure_item_has_latest_revision_id(no_revision_item)
assert not result
assert not db.session.query(Item).filter(
Item.name == no_revision_item.name).first()
# Test case #2: Create two item revisions for the given item, but don't attach them to the item.
# After the fixer runs, it should return the item with proper hashes and a proper
# link to the latest version.
ds = Datastore()
no_revision_item = Item(region="us-east-1",
name="NOREVISION",
account_id=self.account.id,
tech_id=self.technology.id)
db.session.add(no_revision_item)
db.session.commit()
ir_one = ItemRevision(config=ACTIVE_CONF,
date_created=datetime.datetime.utcnow(),
item_id=no_revision_item.id)
ir_two = ItemRevision(config=ACTIVE_CONF,
date_created=(datetime.datetime.utcnow() -
timedelta(days=1)),
item_id=no_revision_item.id)
db.session.add(ir_one)
db.session.add(ir_two)
db.session.commit()
assert len(
db.session.query(ItemRevision).filter(
ItemRevision.item_id == no_revision_item.id).all()) == 2
result = ensure_item_has_latest_revision_id(no_revision_item)
assert result
assert result.latest_revision_id == ir_one.id
assert ds.hash_config(
ACTIVE_CONF) == no_revision_item.latest_revision_complete_hash
assert ds.durable_hash(
ACTIVE_CONF, watcher.ephemeral_paths
) == no_revision_item.latest_revision_durable_hash
# Undo the mock:
security_monkey.watcher.watcher_registry = old_watcher_registry
def pre_test_setup(self):
ResourcePolicyAuditor(accounts=['TEST_ACCOUNT']).OBJECT_STORE.clear()
account_type_result = AccountType(name='AWS')
db.session.add(account_type_result)
db.session.commit()
# main
account = Account(identifier="012345678910",
name="TEST_ACCOUNT",
account_type_id=account_type_result.id,
notes="TEST_ACCOUNT",
third_party=False,
active=True)
# friendly
account2 = Account(identifier="222222222222",
name="TEST_ACCOUNT_TWO",
account_type_id=account_type_result.id,
notes="TEST_ACCOUNT_TWO",
third_party=False,
active=True)
# third party
account3 = Account(identifier="333333333333",
name="TEST_ACCOUNT_THREE",
account_type_id=account_type_result.id,
notes="TEST_ACCOUNT_THREE",
third_party=True,
active=True)
db.session.add(account)
db.session.add(account2)
db.session.add(account3)
db.session.commit()
datastore = Datastore()
# S3
datastore.store('s3',
'us-east-1',
'TEST_ACCOUNT',
'my-test-s3-bucket',
True,
dict(),
arn='arn:aws:s3:::my-test-s3-bucket')
datastore.store('s3',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-s3-bucket-two',
True,
dict(),
arn='arn:aws:s3:::my-test-s3-bucket-two')
datastore.store('s3',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-s3-bucket-three',
True,
dict(),
arn='arn:aws:s3:::my-test-s3-bucket-three')
# IAM User
datastore.store('iamuser',
'us-east-1',
'TEST_ACCOUNT',
'my-test-iam-user',
True,
dict(UserId='AIDA11111111111111111',
UserName='******'),
arn='arn:aws:iam::012345678910:user/my-test-iam-user')
datastore.store(
'iamuser',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-iam-user-two',
True,
dict(UserId='AIDA22222222222222222',
UserName='******'),
arn='arn:aws:iam::222222222222:user/my-test-iam-user-two')
datastore.store(
'iamuser',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-iam-user-three',
True,
dict(UserId='AIDA33333333333333333',
UserName='******'),
arn='arn:aws:iam::333333333333:user/my-test-iam-user-three')
# IAM Role
datastore.store('iamrole',
'us-east-1',
'TEST_ACCOUNT',
'my-test-iam-role',
True,
dict(RoleId='AISA11111111111111111',
RoleName='my-test-iam-role'),
arn='arn:aws:iam::012345678910:role/my-test-iam-role')
datastore.store(
'iamrole',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-iam-role-two',
True,
dict(RoleId='AISA22222222222222222',
RoleName='my-test-iam-role-two'),
arn='arn:aws:iam::222222222222:role/my-test-iam-role-two')
datastore.store(
'iamrole',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-iam-role-three',
True,
dict(RoleId='AISA33333333333333333',
RoleName='my-test-iam-role-three'),
arn='arn:aws:iam::333333333333:role/my-test-iam-role-three')
# NAT Gateway
datastore.store(
'natgateway',
'us-east-1',
'TEST_ACCOUNT',
'my-test-natgateway',
True,
dict(nat_gateway_addresses=[
dict(public_ip='54.11.11.11', private_ip='172.16.11.11')
]),
arn=None) # natgateway has no ARN :(
datastore.store(
'natgateway',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-natgateway-two',
True,
dict(nat_gateway_addresses=[
dict(public_ip='54.22.22.22', private_ip='172.16.22.22')
]),
arn=None) # natgateway has no ARN :(
datastore.store(
'natgateway',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-natgateway-three',
True,
dict(nat_gateway_addresses=[
dict(public_ip='54.33.33.33', private_ip='172.16.33.33')
]),
arn=None) # natgateway has no ARN :(
# VPC
datastore.store(
'vpc',
'us-east-1',
'TEST_ACCOUNT',
'my-test-vpc',
True,
dict(id='vpc-11111111', cidr_block='10.1.1.1/18'),
arn='arn:aws:ec2:us-east-1:012345678910:vpc/vpc-11111111')
datastore.store(
'vpc',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-vpc-two',
True,
dict(id='vpc-22222222', cidr_block='10.2.2.2/18'),
arn='arn:aws:ec2:us-east-1:222222222222:vpc/vpc-22222222')
datastore.store(
'vpc',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-vpc-three',
True,
dict(id='vpc-33333333', cidr_block='10.3.3.3/18'),
arn='arn:aws:ec2:us-east-1:333333333333:vpc/vpc-33333333')
# VPC Service Endpoint (For S3 and things)
datastore.store('endpoint',
'us-east-1',
'TEST_ACCOUNT',
'my-test-vpce',
True,
dict(id='vpce-11111111'),
arn=None) # vpce has no ARN :(
datastore.store('endpoint',
'us-east-1',
'TEST_ACCOUNT_TWO',
'my-test-vpce-two',
True,
dict(id='vpce-22222222'),
arn=None) # vpce has no ARN :(
datastore.store('endpoint',
'us-east-1',
'TEST_ACCOUNT_THREE',
'my-test-vpce-three',
True,
dict(id='vpce-33333333'),
arn=None) # vpce has no ARN :(
def test_ensure_item_has_latest_revision_id(self):
"""
Test that items always have a proper current revision set. Otherwise, the item needs to be deleted.
:return:
"""
from security_monkey.watchers.iam.iam_role import IAMRole
from security_monkey.watcher import ensure_item_has_latest_revision_id
from security_monkey.datastore import Datastore
# Stop the watcher registry from stepping on everyone's toes:
import security_monkey.watcher
old_watcher_registry = security_monkey.watcher.watcher_registry
security_monkey.watcher.watcher_registry = {IAMRole.index: IAMRole}
# Set everything up:
self.setup_batch_db()
watcher = IAMRole(accounts=[self.account.name])
watcher.current_account = (self.account, 0)
watcher.technology = self.technology
# Test case #1: Create an item in the DB that has no current revision ID:
no_revision_item = Item(region="us-east-1", name="NOREVISION", account_id=self.account.id,
tech_id=self.technology.id)
db.session.add(no_revision_item)
db.session.commit()
assert db.session.query(Item).filter(Item.name == no_revision_item.name).one()
# Should delete the item from the DB:
result = ensure_item_has_latest_revision_id(no_revision_item)
assert not result
assert not db.session.query(Item).filter(Item.name == no_revision_item.name).first()
# Test case #2: Create two item revisions for the given item, but don't attach them to the item.
# After the fixer runs, it should return the item with proper hashes and a proper
# link to the latest version.
ds = Datastore()
no_revision_item = Item(region="us-east-1", name="NOREVISION", account_id=self.account.id,
tech_id=self.technology.id)
db.session.add(no_revision_item)
db.session.commit()
ir_one = ItemRevision(config=ACTIVE_CONF, date_created=datetime.datetime.utcnow(),
item_id=no_revision_item.id)
ir_two = ItemRevision(config=ACTIVE_CONF,
date_created=(datetime.datetime.utcnow() - timedelta(days=1)),
item_id=no_revision_item.id)
db.session.add(ir_one)
db.session.add(ir_two)
db.session.commit()
assert len(db.session.query(ItemRevision).filter(ItemRevision.item_id == no_revision_item.id).all()) == 2
result = ensure_item_has_latest_revision_id(no_revision_item)
assert result
assert result.latest_revision_id == ir_one.id
assert ds.hash_config(ACTIVE_CONF) == no_revision_item.latest_revision_complete_hash
assert ds.durable_hash(ACTIVE_CONF, watcher.ephemeral_paths) == no_revision_item.latest_revision_durable_hash
# Undo the mock:
security_monkey.watcher.watcher_registry = old_watcher_registry