def executable(self):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> mock.mock(None, 'which', None)
>>> Serverless("path/to/project").executable
Traceback (most recent call last):
SystemExit: 2
>>> mock.calls_for('eprint')
"error: could not find framework executable: '{}', try using --framework-path", 'serverless'
>>> mock.mock(None, 'which', "/usr/bin/serverless")
>>> Serverless("path/to/project").executable
'/usr/bin/serverless'
"""
if not hasattr(self, '_executable'):
executable = which(self.executable_name)
if not executable:
eprint(
"error: could not find framework executable: '{}', try using --framework-path",
self.executable_name)
raise SystemExit(2)
self._executable = os.path.abspath(executable)
return self._executable
def get_function_root(self, name):
self.package()
package_name = self._get_function_package_name(name)
function_root = os.path.join(self.serverless_package.name,
package_name)
if os.path.exists(function_root):
return function_root
try:
zipfile = ZipFile(
os.path.join(self.serverless_package.name,
"{}.zip".format(package_name)), 'r')
except FileNotFoundError:
eprint(
"error: serverless package did not create a function zip for '{}'",
name)
raise SystemExit(2)
except BadZipFile:
eprint(
"error: serverless package did not create a valid function zip for '{}'",
name)
raise SystemExit(2)
with zipfile:
zipfile.extractall(function_root)
return function_root
def check_version():
try:
response = request.urlopen(
"http://cli.puresec.io/verify/version/{}".format(
puresec_cli.__version__))
except urllib.error.URLError:
return
try:
response = json.loads(response.read().decode())
except ValueError:
return
if not isinstance(response, dict):
return
try:
is_uptodate, last_version = response['is_uptodate'], response[
'last_version']
except KeyError:
return
if not is_uptodate:
eprint(
"warn: you are using an outdated version of PureSec CLI (installed={}, latest={})"
.format(puresec_cli.__version__, last_version))
def get_function_name(self, provider_function_name):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> framework = ServerlessFramework("path/to/project", {})
>>> framework._serverless_config_cache = {'service': {'functions': {'otherFunction': {'name': 'other-function'}}}}
>>> framework.get_function_name('function-name')
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint')
"error: could not find Serverless name for function: '{}'", 'function-name'
>>> framework._serverless_config_cache = {'service': {'functions': {'functionName': {'name': 'function-name'}}}}
>>> framework.get_function_name('function-name')
'functionName'
"""
for name, function_config in self.serverless_config['service'].get(
'functions', {}).items():
if function_config['name'] == provider_function_name:
return name
eprint("error: could not find Serverless name for function: '{}'",
provider_function_name)
raise SystemExit(-1)
def get_function_root(self, name):
if not hasattr(self, 'functions_output'):
self.functions_output = TemporaryDirectory(
"puresec-serverless-functions-")
package_name = self._get_function_package_name(name)
function_root = os.path.join(self.functions_output.name, package_name)
if os.path.exists(function_root):
return function_root
try:
zipfile = ZipFile(
os.path.join(self.serverless_package,
"{}.zip".format(package_name)), 'r')
except FileNotFoundError:
eprint(
"error: serverless package did not create a function zip for '{}'",
name)
raise SystemExit(2)
except BadZipFile:
eprint(
"error: serverless package did not create a valid function zip for '{}'",
name)
raise SystemExit(2)
with zipfile:
zipfile.extractall(function_root)
return function_root
def serverless_config(self):
"""
>>> from pprint import pprint
>>> from collections import namedtuple
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> TemporaryDirectory = namedtuple('TemporaryDirectory', ('name',))
>>> serverless = Serverless("path/to/project")
>>> serverless.package = lambda: None
>>> serverless.serverless_package = TemporaryDirectory('/tmp/package')
>>> serverless.serverless_config
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint')
'error: serverless package did not create serverless-state.json'
>>> with mock.open('/tmp/package/serverless-state.json', 'w') as f:
... f.write('invalid') and None
>>> serverless.serverless_config
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint') # ValueError for <=3.4, JSONDecodeError for >=3.5
'error: invalid serverless-state.json:\\n{}', ...Error('Expecting value: line 1 column 1 (char 0)',)
>>> with mock.open('/tmp/package/serverless-state.json', 'w') as f:
... f.write('{ "x": { "y": 1 }, "z": 2 }') and None
>>> pprint(serverless.serverless_config)
{'x': {'y': 1}, 'z': 2}
"""
if hasattr(self, '_serverless_config_cache'):
return self._serverless_config_cache
self.package()
try:
serverless_config = open(os.path.join(self.serverless_package.name,
'serverless-state.json'),
'r',
errors='replace')
except FileNotFoundError:
eprint(
"error: serverless package did not create serverless-state.json"
)
raise SystemExit(-1)
with serverless_config:
try:
self._serverless_config_cache = json.load(serverless_config)
except ValueError as e:
eprint("error: invalid serverless-state.json:\n{}", e)
raise SystemExit(-1)
return self._serverless_config_cache
def default_account(self):
if not hasattr(self, '_default_account'):
try:
self._default_account = self.session.client(
'sts').get_caller_identity()['Account']
except (botocore.exceptions.BotoCoreError,
botocore.exceptions.ClientError) as e:
eprint("error: failed to get account from aws:\n{}", e)
raise SystemExit(-1)
return self._default_account
def get_cached_api_result(self, service, region, account, api_method, api_kwargs={}):
client = self.get_client(service, region, account)
if client is None:
eprint("error: cannot create {} client for region: '{}', account: '{}'", service, region, account)
return
cache_key = (client, api_method, frozenset(api_kwargs.items()))
result = AwsApi.RESOURCE_CACHE.get(cache_key)
if result is None:
try:
result = AwsApi.RESOURCE_CACHE[cache_key] = getattr(client, api_method)(**api_kwargs)
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
eprint("error: failed to list resources on {}:\n{}", service, e)
raise SystemExit(-1)
return result
def serverless_package(self):
if self.args.framework_output:
return self.args.framework_output
if not hasattr(self, '_package'):
# sanity check so that we know FileNotFoundError later means Serverless is not installed
serverless_config_path = os.path.join(self.path, "serverless.yml")
if not os.path.exists(serverless_config_path):
eprint("error: could not find serverless config in: {}",
serverless_config_path)
raise SystemExit(-1)
self._package = TemporaryDirectory(
prefix="puresec-serverless-package-")
try:
# Suppressing output
subprocess.check_output(
['serverless', 'package', '--package', self._package.name],
cwd=self.path,
stderr=subprocess.STDOUT)
except FileNotFoundError:
eprint(
"error: serverless framework not installed, run `npm install -g severless` (or use --framework-path if not globally installed)"
)
raise SystemExit(-1)
except subprocess.CalledProcessError as e:
eprint("error: serverless package failed:\n{}",
e.output.decode())
raise SystemExit(-1)
return self._package.name
def cloudformation_template(self):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> Aws(resource_template="path/to/cloudformation.json").cloudformation_template
Traceback (most recent call last):
SystemExit: 2
>>> mock.calls_for('eprint')
'error: could not find CloudFormation template in: {}', 'path/to/cloudformation.json'
>>> with mock.open("path/to/cloudformation.json", 'w') as f:
... f.write("not a JSON") and None
>>> Aws(resource_template="path/to/cloudformation.json").cloudformation_template
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint') # ValueError for <=3.4, JSONDecodeError for >=3.5
'error: invalid CloudFormation template:\\n{}', ...Error('Expecting value: line 1 column 1 (char 0)',)
>>> with mock.open("path/to/cloudformation.json", 'w') as f:
... f.write('{ "a": { "b": 1 } }') and None
>>> Aws(resource_template="path/to/cloudformation.json").cloudformation_template
{'a': {'b': 1}}
"""
if not hasattr(self, '_cloudformation_template'):
if not self.resource_template:
self._cloudformation_template = None
self.cloudformation_filetype = None
return
_, self.cloudformation_filetype = os.path.splitext(
self.resource_template)
if self.cloudformation_filetype not in Aws.TEMPLATE_LOADERS:
eprint("error: don't know how to load '{}' file",
self.resource_template)
raise SystemExit(2)
try:
resource_template = open(self.resource_template,
'r',
errors='replace')
except FileNotFoundError:
eprint("error: could not find CloudFormation template in: {}",
self.resource_template)
raise SystemExit(2)
with resource_template:
try:
self._cloudformation_template = Aws.TEMPLATE_LOADERS[
self.cloudformation_filetype](
resource_template,
default_region=self.default_region)
except ValueError as e:
eprint("error: invalid CloudFormation template:\n{}", e)
raise SystemExit(-1)
return self._cloudformation_template
def session(self):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> class Framework:
... def get_default_profile(self):
... return "default_profile"
>>> with mock.open("path/to/cloudformation.json", 'w') as f:
... f.write('{"a": {"b": 1}}') and None
>>> Aws(resource_template="path/to/cloudformation.json").session
Session(...)
>>> Aws(resource_template="path/to/cloudformation.json", framework=Framework()).session
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint')
'error: failed to create aws session:\\n{}', ProfileNotFound('The config profile (default_profile) could not be found',)
"""
if not hasattr(self, '_session'):
if self.framework:
profile = self.framework.get_default_profile()
else:
profile = None
try:
self._session = boto3.Session(profile_name=profile)
except (botocore.exceptions.BotoCoreError,
botocore.exceptions.ClientError) as e:
eprint("error: failed to create aws session:\n{}", e)
raise SystemExit(-1)
return self._session
def __init__(self, path, config, resource_template=None, runtime=None, handler=None, function_name=None, framework=None, function=None, args=None):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> AwsProvider("path/to/project", config={})
Traceback (most recent call last):
SystemExit: 2
>>> mock.calls_for('eprint')
'error: must supply either --resource-template, --runtime, or --framework'
>>> with mock.open("path/to/cloudformation.json", 'w') as f:
... f.write('{}') and None
>>> AwsProvider("path/to/project", config={}, resource_template="path/to/cloudformation.json", runtime='nodejs') and None
>>> mock.calls_for('eprint')
'warn: ignoring --runtime when --resource-template or --framework supplied'
"""
Base.__init__(
self,
path, config,
resource_template=resource_template,
runtime=runtime,
handler=handler,
function_name=function_name,
framework=framework,
function=function,
args=args
)
Aws.__init__(
self,
resource_template=self.resource_template,
framework=self.framework,
)
if not self.resource_template and not self.runtime:
eprint("error: must supply either --resource-template, --runtime, or --framework")
raise SystemExit(2)
if self.resource_template:
if self.runtime:
eprint("warn: ignoring --runtime when --resource-template or --framework supplied")
if self.handler:
eprint("warn: ignoring --handler when --resource-template or --framework supplied")
if self.function_name:
eprint("warn: ignoring --function-name when --resource-template or --framework supplied")
def generate_provider(self, path, framework, config):
if framework:
provider = framework.get_provider_name()
if provider:
if self.args.provider:
# self.args.provider always in providers.__all__, no need to check
if self.args.provider != provider:
eprint(
"error: conflict between --provider ('{}') option and framework ('{}')"
.format(self.args.provider, provider))
raise SystemExit(2)
elif provider not in providers.__all__:
eprint("error: provider not yet supported: '{}'".format(
provider))
raise SystemExit(2)
else:
if not self.args.provider:
eprint(
"error: could not determine provider from framework, please specify with --provider"
)
raise SystemExit(2)
provider = self.args.provider
else:
if not self.args.provider:
eprint("error: must specify either --provider or --framework")
raise SystemExit(2)
provider = self.args.provider
provider = import_module(
"puresec_cli.actions.generate_roles.providers.{}".format(
provider)).Provider(
path,
config,
resource_template=self.args.resource_template,
runtime=self.args.runtime,
handler=self.args.handler,
function_name=self.args.function_name,
framework=framework,
function=self.args.function,
args=self.args,
)
with provider:
yield provider
def package(self):
"""
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> Serverless("path/to/project").package()
Traceback (most recent call last):
SystemExit: -1
>>> mock.calls_for('eprint')
'error: could not find serverless config in: {}', 'path/to/project/serverless.yml'
"""
if not hasattr(self, 'serverless_package'):
# sanity check so that we know FileNotFoundError later means Serverless is not installed
serverless_config_path = os.path.join(self.path, "serverless.yml")
if not os.path.exists(serverless_config_path):
eprint("error: could not find serverless config in: {}",
serverless_config_path)
raise SystemExit(-1)
self.serverless_package = TemporaryDirectory(prefix="puresec-")
try:
# Suppressing output
subprocess.check_output([
self.executable, 'package', '--package',
self.serverless_package.name
],
cwd=self.path,
stderr=subprocess.STDOUT)
except FileNotFoundError:
eprint(
"error: serverless framework not installed, run `npm install -g severless` (or use --framework-path if not globally installed)"
)
raise SystemExit(-1)
except subprocess.CalledProcessError as e:
eprint("error: serverless package failed:\n{}",
e.output.decode())
raise SystemExit(-1)
def _walk(self, processor, *args, **kwargs):
"""
>>> from collections import namedtuple
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(pkg_resources, 'resource_filename', "/path/to/node_modules")
>>> processed = []
>>> def processor(filename, contents, custom_positional, custom_keyword):
... processed.append((filename, contents, custom_positional, custom_keyword))
>>> def stat(self, filename):
... return namedtuple('Stat', ('st_size',))(5*1024*1024 if filename == "/path/to/function/large-file" else 512)
>>> mock.mock(NodejsRuntime, '_stat', stat)
>>> mock.filesystem = {'': {'path': {'to': {'function': {
... 'large-file': True,
... 'config': True,
... 'unreferenced': True,
... }}}}}
>>> with mock.open("/path/to/function/config", 'w') as f:
... f.write("some config") and None
>>> with mock.open("/path/to/function/src/index.js", 'w') as f:
... f.write("some code config large-file more code") and None
>>> mock.mock(subprocess, 'check_output', b"/path/to/function/src/index.js\\n")
>>> NodejsRuntime('/path/to/function', resource_properties={'Handler': "src/index.handler"}, provider=object()) \\
... ._walk(processor, 'positional', custom_keyword='keyword')
>>> mock.calls_for('subprocess.check_output')
['node', '/path/to/node_modules/dependency-tree/bin/cli.js', '/path/to/function/src/index.js', '--directory', '/path/to/function', '--list-form'], stderr=-2
>>> processed
[('/path/to/function/src/index.js', 'some code config large-file more code', 'positional', 'keyword'),
('/path/to/function/config', 'some config', 'positional', 'keyword')]
"""
if hasattr(self, '_dependencies'):
# cached
for filename in self._dependencies:
with open(filename, 'r', errors='replace') as file:
processor(filename, file.read(), *args, **kwargs)
return
# getting main JavaScript file (from Handler)
handler = self.resource_properties.get('Handler')
if not handler:
# dummy CloudFormation? walking everything
super()._walk(processor, *args, **kwargs)
return
module = '.'.join(handler.split(
'.')[0:-1]) # all except the last part which is the method
filename = os.path.abspath(
os.path.join(self.root, "{}.js".format(module)))
if not os.path.exists(filename):
return
# acquiring dependencies using NPM's dependency-tree
dependency_tree_cli_path = os.path.abspath(
os.path.join(
pkg_resources.resource_filename('puresec_cli',
'resources/node_modules'),
'dependency-tree/bin/cli.js'))
try:
dependencies = subprocess.check_output([
'node', dependency_tree_cli_path, filename, '--directory',
self.root, '--list-form'
],
stderr=subprocess.STDOUT)
except FileNotFoundError:
eprint("error: function runtime (nodejs) must be installed")
raise SystemExit(-1)
except subprocess.CalledProcessError as e:
eprint("error: failed to get dependency tree:\n{}",
e.output.decode())
raise SystemExit(-1)
dependencies = [
dependency for dependency in dependencies.decode().split('\n')
# skipping last blank line and aws-sdk
if dependency and '/node_modules/aws-sdk/' not in dependency
]
self._dependencies = dependencies[:] # cache
# getting all non-dependency files
resources = [] # (abspath, filename)
for path, dirs, filenames in os.walk(self.root):
if path.endswith('node_modules'):
# skipping aws-sdk
try:
dirs.remove('aws-sdk')
except ValueError:
pass
paths_generator = (
(os.path.abspath(os.path.join(path, filename)), filename)
for filename in filenames if
not NodejsRuntime.JAVASCRIPT_FILENAME_PATTERN.search(filename))
resources.extend(paths_tuple for paths_tuple in paths_generator
if self._stat(paths_tuple[0]).st_size <
NodejsRuntime.MAX_FILE_SIZE)
while dependencies:
filename = dependencies.pop(0)
with open(filename, 'r', errors='replace') as file:
# adding resources referenced by current file
used_resources_indexes = []
contents = file.read()
for index, (resource_abspath,
resource_filename) in enumerate(resources):
if resource_filename in contents:
dependencies.append(resource_abspath)
self._dependencies.append(resource_abspath)
used_resources_indexes.append(index)
for index in reversed(used_resources_indexes):
resources.pop(index)
# processing current file
processor(filename, contents, *args, **kwargs)
def _normalize_actions(self, resources, parents):
""" Convert set to match-all when there's at least one.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> resources = {'table/sometable': {'a', 'b', 'c'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'a', 'b', 'c'}}
>>> resources = {'table/sometable': {'a', '*', 'c'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}}
>>> resources = {'table/sometable': set(), 'table/sometable/stream/somestream': {'dynamodb:DescribeStream'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable/stream/somestream': {'dynamodb:DescribeStream'}}
>>> resources = {'table/sometable/stream/somestream': set(), 'table/sometable': {'dynamodb:GetItem'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'dynamodb:GetItem'}}
>>> mock.mock(None, 'eprint')
>>> resources = {'table/sometable': set()}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> mock.calls_for('eprint')
"warn: unknown actions for '{}:{}', couldn't find any relevant SDK methods in your code, falling back to '*'", 'dynamodb:us-west-1', 'table/sometable'
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}}
>>> resources = {'table/sometable': set(), 'table/sometable/stream/somestream': set()}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}, 'table/sometable/stream/somestream': {'*'}}
"""
for resource, actions in tuple(resources.items()):
if not actions:
# if there are no other resources with common name that *do* have actions
if any((other_resource.startswith(resource.rstrip('*'))
or resource.startswith(other_resource.rstrip('*')))
and other_actions.difference({'*'})
for other_resource, other_actions in resources.items()):
# then it's fine
del resources[resource]
else:
actions.add('*')
eprint(
"warn: unknown actions for '{}:{}', couldn't find any relevant SDK methods in your code, falling back to '*'",
':'.join(parents), resource)
elif '*' in actions:
actions.clear()
actions.add('*')
def _normalize_resources(self, resources, parents):
""" Convert dict to match-all when there's at least one.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> resources = {'a': {'x'}, 'b': {'y'}, 'c': {'z'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(resources)
{'a': {'x'}, 'b': {'y'}, 'c': {'z'}}
>>> resources = {'a': {'x'}, '*': {'y'}, 'c': {'z'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'*': {'x', 'y', 'z'}}
>>> resources = {'activity/b': {'z'}, 'execution/*': {'*'}, 'stateMachine/a': {'x'}, 'stateMachine/*': {'y'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'activity/b': {'z'}, 'execution/*': {'*'}, 'stateMachine/*': {'x', 'y'}}
>>> mock.mock(None, 'eprint')
>>> resources = defaultdict(set)
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> mock.calls_for('eprint')
"warn: unknown resources for '{}', couldn't find anything relevant in your AWS account or CloudFormation, falling back to '*'", 'dynamodb:us-west-1'
>>> dict(resources)
{'*': set()}
"""
if not resources:
resources['*'] # accessing to initialize defaultdict
eprint(
"warn: unknown resources for '{}', couldn't find anything relevant in your AWS account or CloudFormation, falling back to '*'",
':'.join(parents))
else:
# mapping all resources wildcard matching to others
wildcard_matches = {}
for resource in resources.keys():
if '*' in resource or '?' in resource:
matches = fnmatch.filter(resources.keys(), resource)
if len(matches) > 1: # not just self
wildcard_matches[resource] = matches
for wildcard, matches in wildcard_matches.items():
if wildcard not in resources:
continue # processed by another resource
merged = set()
for resource in matches:
if resource not in resources:
continue # processed by another resource
merged.update(resources[resource])
del resources[resource]
resources[wildcard] = merged
class Base(RuntimeBase, BaseApi):
__metaclass__ = abc.ABCMeta
def __init__(self, root, resource_properties, provider):
super().__init__(root, provider)
self.resource_properties = resource_properties
self.environment_variables = self.resource_properties.get(
'Environment', {}).get('Variables', {})
# { service: { region: { account: { resource: {action} } } } }
self._permissions = defaultdict(
lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(set))))
@property
def permissions(self):
"""
>>> from pprint import pprint
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> runtime._permissions = {
... 'dynamodb': {'us-west-1': {'111': {'table/a': {'GetRecords'}, 'table/b': {'UpdateItem'}}}},
... 'ses': {'*': {'111': {'*': {'*'}}, '222': {'*': {'*'}}}},
... }
>>> pprint(runtime.permissions)
{'arn:aws:dynamodb:us-west-1:111:table/a': {'GetRecords'},
'arn:aws:dynamodb:us-west-1:111:table/b': {'UpdateItem'},
'arn:aws:ses:*:111:*': {'*'},
'arn:aws:ses:*:222:*': {'*'}}
"""
permissions = {}
for service, regions in self._permissions.items():
for region, accounts in regions.items():
for account, resources in accounts.items():
for resource, actions in resources.items():
permissions["arn:aws:{}:{}:{}:{}".format(
service, region, account, resource)] = actions
return permissions
# Processing (override these)
def process(self):
self._process_services()
self._process_regions()
self._process_resources()
self._process_actions()
self._cleanup()
@abc.abstractmethod
def _get_services(self, filename, contents):
pass
try:
REGION_PATTERNS = dict(
(region, re.compile(r"\b{}\b".format(re.escape(region))))
for region in boto3.Session().get_available_regions('ec2'))
except (botocore.exceptions.BotoCoreError,
botocore.exceptions.ClientError) as e:
eprint("error: failed to create aws session:\n{}", e)
raise SystemExit(-1)
# regions = set()
def _get_regions(self, filename, contents, regions, service, account):
"""
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={'Environment': {'Variables': {'var1': "gigi eu-west-1 laus-east-2", 'var2': "eu-central-1 ca-central-1" }}}, provider=object())
>>> regions = set()
>>> runtime._get_regions('filename', "lalala us-east-1 lululu us-west-1 us-east-2la us-west-5 nonono", regions, 'dynamodb', '*')
>>> sorted(regions)
['ca-central-1', 'eu-central-1', 'eu-west-1', 'us-east-1', 'us-west-1']
"""
# From file
regions.update(region
for region, pattern in Base.REGION_PATTERNS.items()
if pattern.search(contents))
# From environment
if not hasattr(self, '_environment_regions'):
self._environment_regions = set(
region for region, pattern in Base.REGION_PATTERNS.items()
if any(
pattern.search(value)
for value in self.environment_variables.values()
if isinstance(value, str)))
regions.update(self._environment_regions)
# resources = defaultdict(set)
@abc.abstractmethod
def _get_resources(self, filename, contents, resources, region, account,
service):
pass
# actions = set()
@abc.abstractmethod
def _get_actions(self, filename, contents, actions, service):
pass
# Sub processors
def _process_services(self):
self._walk(self._get_services)
self._normalize_permissions(self._permissions)
def _process_regions(self):
""" Expands '*' regions to all regions seen within the code.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> mock.mock(Base, '_walk', lambda self, processor, possible_regions, service, account: possible_regions.update({'us-east-1', 'us-east-2'}))
>>> runtime._permissions = {
... 'dynamodb': {'us-west-1': {'111': {'table/a': set(), 'table/b': set()}}},
... 'ses': defaultdict(dict, {'*': {'111': {'*': set()}, '222': {'*': set()}}})
... }
>>> runtime._process_regions()
>>> mock.calls_for('Base._walk')
Runtime, _get_regions, {'us-east-1', 'us-east-2'}, account='111', service='ses'
Runtime, _get_regions, {'us-east-1', 'us-east-2'}, account='222', service='ses'
>>> pprint(normalize_dict(runtime._permissions))
{'dynamodb': {'us-west-1': {'111': {'table/a': set(), 'table/b': set()}}},
'ses': {'us-east-1': {'111': {'*': set()}, '222': {'*': set()}}, 'us-east-2': {'111': {'*': set()}, '222': {'*': set()}}}}
"""
for service, regions in self._permissions.items():
if '*' in regions:
for account, resources in sorted(regions['*'].items()):
possible_regions = set()
self._walk(
self._get_regions,
# custom arguments to processor
possible_regions,
service=service,
account=account)
# moving the account from '*' to possible regions
if possible_regions:
for region in possible_regions:
regions[region][account] = deepcopy(resources)
del regions['*'][account]
if not regions['*']:
# all moved
del regions['*']
def _process_resources(self):
for service, regions in self._permissions.items():
for region, accounts in regions.items():
for account, resources in accounts.items():
self._walk(
self._get_resources,
# custom arguments to processor
resources,
region=region,
account=account,
service=service,
)
self._normalize_resources(resources,
(service, region, account))
def _process_actions(self):
for service, regions in self._permissions.items():
actions = set()
self._walk(
self._get_actions,
# custom arguments to processor
actions,
service=service,
)
for region, accounts in regions.items():
for account, resources in accounts.items():
self._match_resources_actions(service, resources, actions)
self._normalize_actions(resources,
(service, region, account))
# get_all_resources_method: (region, account) => {resource: pattern}
def _get_generic_resources(self, filename, contents, resources, region,
account, resource_format,
get_all_resources_method):
""" Simply greps resources inside the given contents.
>>> from collections import defaultdict
>>> from functools import partial
>>> from pprint import pprint
>>> from tests.mock import Mock
>>> from tests.utils import normalize_dict
>>> mock = Mock(__name__)
>>> class Provider:
... pass
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={'Environment': {'Variables': {'var1': "gigi table-1 latable-6", 'var2': "table-2 table-3"}}}, provider=Provider())
>>> runtime.provider.cloudformation_template = None
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'TableNames': ["table-1", "table-2", "table-3", "table-4", "table-5", "table-6"]})
>>> resources = defaultdict(set)
>>> runtime._get_generic_resources('filename', "lalala table-4 lululu table-5 table-6la table-7 nonono", resources, region='us-east-1', account='some-account',
... resource_format="table/{}", get_all_resources_method=partial(runtime._get_generic_all_resources, 'dynamodb', template_type='AWS::DynamoDB::Table', api_method='list_tables', api_attribute='TableNames'))
>>> pprint(normalize_dict(resources))
{'table/table-1': set(), 'table/table-2': set(), 'table/table-3': set(), 'table/table-4': set(), 'table/table-5': set()}
>>> mock.calls_for('Provider.get_cached_api_result')
'dynamodb', account='some-account', api_kwargs={}, api_method='list_tables', region='us-east-1'
"""
all_resources = get_all_resources_method(region=region,
account=account)
if not all_resources:
resources[resource_format.format('*')]
return
# From file
for resource, pattern in all_resources.items():
if pattern.search(contents):
resources[resource_format.format(resource)]
# From environment
for resource, pattern in all_resources.items():
if any(
pattern.search(value)
for value in self.environment_variables.values()
if isinstance(value, str)):
resources[resource_format.format(resource)]
def _match_resources_actions(self, service, resources, actions):
"""
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> resources = defaultdict(set, {'table/sometable': set(), 'table/sometable/stream/somestream': set()})
>>> actions = {'dynamodb:PutItem', 'dynamodb:GetRecords', 'dynamodb:DeleteItem', 'dynamodb:DescribeStream', 'dynamodb:ListTables'}
>>> runtime._match_resources_actions('dynamodb', resources, actions)
>>> pprint(normalize_dict(resources))
{'*': {'dynamodb:ListTables'}, 'table/sometable': {'dynamodb:DeleteItem', 'dynamodb:PutItem'}, 'table/sometable/stream/somestream': {'dynamodb:DescribeStream', 'dynamodb:GetRecords'}}
>>> resources = defaultdict(set, {'table/sometable': set()})
>>> actions = {'dynamodb:GetRecords', 'dynamodb:DescribeStream'}
>>> runtime._match_resources_actions('dynamodb', resources, actions)
>>> pprint(normalize_dict(resources))
{'table/*/stream/*': {'dynamodb:DescribeStream', 'dynamodb:GetRecords'}, 'table/sometable': set()}
"""
matchers = Base.SERVICE_RESOURCE_ACTION_MATCHERS.get(service)
if not matchers:
if not resources:
resources['*']
# not specific matchers, just add all actions to all resources
for resource, resource_actions in resources.items():
resource_actions.update(actions)
return
unused_actions = set(actions)
# matching with resources
for resource, resource_actions in resources.items():
for pattern, default, matching_actions in matchers:
if pattern.match(resource):
matching = matching_actions.intersection(actions)
unused_actions.difference_update(matching)
resource_actions.update(matching)
break
# adding unused actions to 'default' resources
for action in unused_actions:
for pattern, default, matching_actions in matchers:
if action in matching_actions:
resources[default].add(action)
def _cleanup(self):
""" Merges region-less services and resource-less actions.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> runtime._permissions = {
... 'dynamodb': {'us-west-1': {'111': {'table/a': {'dynamodb:GetItem'}}}},
... 's3': defaultdict(dict, {'us-east-1': {'some-account': {'somebucket': {'s3:CreateBucket'}}}, 'us-west-1': {'another-account': {'anotherbucket': {'s3:ListObjects'}}}})
... }
>>> runtime._cleanup()
>>> pprint(normalize_dict(runtime._permissions))
{'dynamodb': {'us-west-1': {'111': {'table/a': {'dynamodb:GetItem'}}}},
's3': {'': {'another-account': {'anotherbucket': {'s3:ListObjects'}}, 'some-account': {'*': {'s3:CreateBucket'}}}}}
"""
# Region-less services
for service in Base.REGIONLESS_SERVICES:
if service not in self._permissions:
continue
merged = reduce(deepmerge, self._permissions[service].values())
self._permissions[service].clear()
self._permissions[service][''] = merged
for service, resourceless_actions in Base.SERVICE_RESOURCELESS_ACTIONS.items(
):
if service not in self._permissions:
continue
for region, accounts in self._permissions[service].items():
for account, resources in accounts.items():
found_actions = set()
for resource, actions in tuple(resources.items()):
for action in resourceless_actions:
if action in actions:
found_actions.add(action)
actions.remove(action)
if not actions:
del resources[resource]
if found_actions:
resources['*'] = found_actions
# Helpers
def _normalize_permissions(self, tree):
""" Merge trees when one of the keys have '*' permission.
>>> from pprint import pprint
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> tree = {'a': {'b': {'c': 1}, '*': {'d': 2}, 'e': {'f': 3}}}
>>> runtime._normalize_permissions(tree)
>>> pprint(tree)
{'a': {'*': {'c': 1, 'd': 2, 'f': 3}}}
>>> tree = {'b': {'c': 1}, '*': {'d': 2}, 'e': {'f': 3}}
>>> runtime._normalize_permissions(tree)
>>> pprint(tree)
{'*': {'c': 1, 'd': 2, 'f': 3}}
"""
if '*' in tree:
merged = reduce(deepmerge, tree.values())
tree.clear()
tree['*'] = merged
for k, v in tree.items():
if isinstance(v, dict):
self._normalize_permissions(v)
def _normalize_resources(self, resources, parents):
""" Convert dict to match-all when there's at least one.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> resources = {'a': {'x'}, 'b': {'y'}, 'c': {'z'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(resources)
{'a': {'x'}, 'b': {'y'}, 'c': {'z'}}
>>> resources = {'a': {'x'}, '*': {'y'}, 'c': {'z'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'*': {'x', 'y', 'z'}}
>>> resources = {'activity/b': {'z'}, 'execution/*': {'*'}, 'stateMachine/a': {'x'}, 'stateMachine/*': {'y'}}
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'activity/b': {'z'}, 'execution/*': {'*'}, 'stateMachine/*': {'x', 'y'}}
>>> mock.mock(None, 'eprint')
>>> resources = defaultdict(set)
>>> runtime._normalize_resources(resources, ['dynamodb', 'us-west-1'])
>>> mock.calls_for('eprint')
"warn: unknown resources for '{}', couldn't find anything relevant in your AWS account or CloudFormation, falling back to '*'", 'dynamodb:us-west-1'
>>> dict(resources)
{'*': set()}
"""
if not resources:
resources['*'] # accessing to initialize defaultdict
eprint(
"warn: unknown resources for '{}', couldn't find anything relevant in your AWS account or CloudFormation, falling back to '*'",
':'.join(parents))
else:
# mapping all resources wildcard matching to others
wildcard_matches = {}
for resource in resources.keys():
if '*' in resource or '?' in resource:
matches = fnmatch.filter(resources.keys(), resource)
if len(matches) > 1: # not just self
wildcard_matches[resource] = matches
for wildcard, matches in wildcard_matches.items():
if wildcard not in resources:
continue # processed by another resource
merged = set()
for resource in matches:
if resource not in resources:
continue # processed by another resource
merged.update(resources[resource])
del resources[resource]
resources[wildcard] = merged
def _normalize_actions(self, resources, parents):
""" Convert set to match-all when there's at least one.
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(Base):
... pass
>>> runtime = Runtime('path/to/function', resource_properties={}, provider=object())
>>> resources = {'table/sometable': {'a', 'b', 'c'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'a', 'b', 'c'}}
>>> resources = {'table/sometable': {'a', '*', 'c'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}}
>>> resources = {'table/sometable': set(), 'table/sometable/stream/somestream': {'dynamodb:DescribeStream'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable/stream/somestream': {'dynamodb:DescribeStream'}}
>>> resources = {'table/sometable/stream/somestream': set(), 'table/sometable': {'dynamodb:GetItem'}}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'dynamodb:GetItem'}}
>>> mock.mock(None, 'eprint')
>>> resources = {'table/sometable': set()}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> mock.calls_for('eprint')
"warn: unknown actions for '{}:{}', couldn't find any relevant SDK methods in your code, falling back to '*'", 'dynamodb:us-west-1', 'table/sometable'
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}}
>>> resources = {'table/sometable': set(), 'table/sometable/stream/somestream': set()}
>>> runtime._normalize_actions(resources, ['dynamodb', 'us-west-1'])
>>> pprint(normalize_dict(resources))
{'table/sometable': {'*'}, 'table/sometable/stream/somestream': {'*'}}
"""
for resource, actions in tuple(resources.items()):
if not actions:
# if there are no other resources with common name that *do* have actions
if any((other_resource.startswith(resource.rstrip('*'))
or resource.startswith(other_resource.rstrip('*')))
and other_actions.difference({'*'})
for other_resource, other_actions in resources.items()):
# then it's fine
del resources[resource]
else:
actions.add('*')
eprint(
"warn: unknown actions for '{}:{}', couldn't find any relevant SDK methods in your code, falling back to '*'",
':'.join(parents), resource)
elif '*' in actions:
actions.clear()
actions.add('*')
def get_client(self, service, region, account):
"""
>>> from pprint import pprint
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Session:
... def client(self, *args, **kwargs):
... return (args, kwargs)
>>> provider = AwsApi()
>>> provider.config = {}
>>> provider.session = Session()
>>> provider.default_account = 'default_account'
>>> class Args:
... pass
>>> args = Args()
>>> args.no_input = False
>>> provider.args = args
>>> pprint(provider.get_client('dynamodb', 'us-east-1', 'default_account'))
(('dynamodb',), {'region_name': 'us-east-1'})
>>> mock.mock(None, 'eprint')
>>> pprint(provider.get_client('dynamodb', 'us-east-1', '*'))
(('dynamodb',), {'region_name': 'us-east-1'})
>>> mock.calls_for('eprint')
"warn: unknown account ('*'), using default session"
>>> class Session:
... def client(self, *args, **kwargs):
... return (args, kwargs)
>>> mock.mock(boto3, 'Session', Session())
>>> mock.mock(None, 'input', lambda message: 'dummy')
>>> pprint(provider.get_client('dynamodb', 'us-east-1', 'another_account'))
(('dynamodb',), {'region_name': 'us-east-1'})
>>> mock.calls_for('boto3.Session')
profile_name='dummy'
>>> pprint(provider.config)
{'aws': {'accounts': {'another_account': {'profile': 'dummy'}}}}
"""
client = AwsApi.CLIENTS_CACHE.get((service, region, account))
if client:
return client # from cache
if region == '*':
eprint("warn: unknown region ('*'), using the default ('{}')", self.default_region)
region = self.default_region
if account == '*':
eprint("warn: unknown account ('*'), using default session")
client = self.session.client(
service,
region_name=region
)
elif account == self.default_account:
client = self.session.client(
service,
region_name=region
)
elif self.args.no_input:
eprint("warn: unknown account ('{}') and --no-input set, using default session", account)
client = self.session.client(
service,
region_name=region
)
else:
account_config = self.config.setdefault('aws', {}).setdefault('accounts', {}).setdefault(account, {})
if not 'profile' in account_config:
account_config['profile'] = input("Enter configured AWS profile for {}: ".format(account))
client = boto3.Session(profile_name=account_config['profile']).client(service, region_name=region)
AwsApi.CLIENTS_CACHE[(service, region, account)] = client
return client
def _walk(self, processor, *args, **kwargs):
"""
>>> from collections import namedtuple
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(pkg_resources, 'resource_filename', "/path/to/list-dependencies.py")
>>> processed = []
>>> def processor(filename, contents, custom_positional, custom_keyword):
... processed.append((filename, contents, custom_positional, custom_keyword))
>>> def stat(self, filename):
... return namedtuple('Stat', ('st_size',))(5*1024*1024 if filename == "/path/to/function/large-file" else 512)
>>> mock.mock(PythonRuntime, '_stat', stat)
>>> mock.filesystem = {'': {'path': {'to': {'function': {
... 'large-file': True,
... 'config': True,
... 'unreferenced': True,
... }}}}}
>>> with mock.open("/path/to/function/config", 'w') as f:
... f.write("some config") and None
>>> with mock.open("/path/to/function/src/index.py", 'w') as f:
... f.write("some code config large-file more code") and None
>>> mock.mock(subprocess, 'check_output', b"/path/to/function/src/index.py\\n")
>>> PythonRuntime('/path/to/function', resource_properties={'Handler': "src/index.handler", 'Runtime': 'python2.7'}, provider=object()) \\
... ._walk(processor, 'positional', custom_keyword='keyword')
>>> mock.calls_for('subprocess.check_output')
['python2.7', '/path/to/list-dependencies.py', '/path/to/function/src/index.py', '/path/to/function'], stderr=-2
>>> processed
[('/path/to/function/src/index.py', 'some code config large-file more code', 'positional', 'keyword'),
('/path/to/function/config', 'some config', 'positional', 'keyword')]
"""
if hasattr(self, '_dependencies'):
# cached
for filename in self._dependencies:
with open(filename, 'r', errors='replace') as file:
processor(filename, file.read(), *args, **kwargs)
return
# getting main Python file (from Handler)
handler = self.resource_properties.get('Handler')
if not handler:
# dummy CloudFormation? walking everything
super()._walk(processor, *args, **kwargs)
return
module = '.'.join(handler.split(
'.')[0:-1]) # all except the last part which is the method
filename = os.path.abspath(
os.path.join(self.root, "{}.py".format(module.replace('.', '/'))))
if not os.path.exists(filename):
return
# acquiring dependencies with the correct Python version using resources/list-dependencies.py script
list_dependencies_script_path = pkg_resources.resource_filename(
'puresec_cli', 'resources/list-dependencies.py')
python_executable = self.resource_properties[
'Runtime'] # e.g 'python2.7'
try:
dependencies = subprocess.check_output([
python_executable, list_dependencies_script_path, filename,
self.root
],
stderr=subprocess.STDOUT)
except FileNotFoundError:
eprint("error: function runtime ({}) must be installed",
python_executable)
raise SystemExit(-1)
except subprocess.CalledProcessError as e:
eprint("error: failed to get dependency tree:\n{}",
e.output.decode())
raise SystemExit(-1)
dependencies = dependencies.decode().split('\n')
dependencies.pop() # last empty line
self._dependencies = dependencies[:] # cache
# getting all non-dependency files
resources = [] # (abspath, filename)
for path, dirs, filenames in os.walk(self.root):
paths_generator = (
(os.path.abspath(os.path.join(path, filename)), filename)
for filename in filenames
if not PythonRuntime.PYTHON_FILENAME_PATTERN.search(filename))
resources.extend(paths_tuple for paths_tuple in paths_generator
if self._stat(paths_tuple[0]).st_size <
PythonRuntime.MAX_FILE_SIZE)
while dependencies:
filename = dependencies.pop(0)
with open(filename, 'r', errors='replace') as file:
# adding resources referenced by current file
used_resources_indexes = []
contents = file.read()
for index, (resource_abspath,
resource_filename) in enumerate(resources):
if resource_filename in contents:
dependencies.append(resource_abspath)
self._dependencies.append(resource_abspath)
used_resources_indexes.append(index)
for index in reversed(used_resources_indexes):
resources.pop(index)
# processing current file
processor(filename, contents, *args, **kwargs)
def _get_generic_all_resources(self,
service,
region,
account,
template_type,
api_method,
api_attribute,
api_inner_attribute=None,
resource_converter=None,
api_kwargs={},
warn=True):
"""
>>> from pprint import pprint
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Runtime(BaseApi):
... pass
>>> runtime = Runtime()
>>> class Provider:
... pass
>>> runtime.provider = Provider()
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'TableNames': []})
>>> runtime.provider.cloudformation_template = {'Resources': {'T1': {'Type': 'AWS::DynamoDB::Table', 'Properties': {'TableName': 'table-1'}},
... 'T2': {'Type': 'AWS::DynamoDB::Table', 'Properties': {'TableName': 'table-2'}},
... 'B1': {'Type': 'AWS::S3::Bucket', 'Properties': {'TableName': 'not-table-2'}}}}
>>> pprint(runtime._get_generic_all_resources('dynamodb', 'us-east-1', 'some-account', 'AWS::DynamoDB::Table', 'list_tables', 'TableNames'))
{'table-1': re.compile('\\\\btable\\\\-1\\\\b', re.IGNORECASE),
'table-2': re.compile('\\\\btable\\\\-2\\\\b', re.IGNORECASE)}
>>> mock.calls_for('Provider.get_cached_api_result')
'dynamodb', account='some-account', api_kwargs={}, api_method='list_tables', region='us-east-1'
>>> runtime.provider.cloudformation_template = None
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'TableNames': ['table-1', 'table-2']})
>>> pprint(runtime._get_generic_all_resources('dynamodb', 'us-east-1', 'some-account', 'AWS::DynamoDB::Table', 'list_tables', 'TableNames'))
{'table-1': re.compile('\\\\btable\\\\-1\\\\b', re.IGNORECASE),
'table-2': re.compile('\\\\btable\\\\-2\\\\b', re.IGNORECASE)}
>>> mock.calls_for('Provider.get_cached_api_result')
'dynamodb', account='some-account', api_kwargs={}, api_method='list_tables', region='us-east-1'
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'Buckets': [{'Name': "bucket-1"}, {'Name': "bucket-2"}]})
>>> pprint(runtime._get_generic_all_resources('s3', 'us-east-1', 'some-account', 'AWS::S3::Bucket', 'list_buckets', 'Buckets', 'Name'))
{'bucket-1': re.compile('\\\\bbucket\\\\-1\\\\b', re.IGNORECASE),
'bucket-2': re.compile('\\\\bbucket\\\\-2\\\\b', re.IGNORECASE)}
>>> mock.calls_for('Provider.get_cached_api_result')
's3', account='some-account', api_kwargs={}, api_method='list_buckets', region='us-east-1'
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'Topics': [{'TopicArn': "arn:aws:sns:us-east-1:123456789012:my_topic"}]})
>>> pprint(runtime._get_generic_all_resources('sns', 'us-east-1', 'some-account', 'AWS::SNS::Topic', 'list_topics', 'Topics', 'TopicArn',
... resource_converter=lambda topic_arn: BaseApi.ARN_RESOURCE_PATTERN.match(topic_arn).group(1)))
{'my_topic': re.compile('\\\\bmy_topic\\\\b', re.IGNORECASE)}
>>> mock.calls_for('Provider.get_cached_api_result')
'sns', account='some-account', api_kwargs={}, api_method='list_topics', region='us-east-1'
>>> mock.mock(None, 'eprint')
>>> mock.mock(runtime.provider, 'get_cached_api_result', {'TableNames': []})
>>> runtime._get_generic_all_resources('dynamodb', 'us-east-1', 'some-account', 'AWS::DynamoDB::Table', 'list_tables', 'TableNames')
{}
>>> mock.calls_for('eprint')
"warn: no {} resources ({}) on '{}:{}', you're using this service but your AWS account and CloudFormation are empty", 'dynamodb', 'AWS::DynamoDB::Table', 'us-east-1', 'some-account'
>>> mock.calls_for('Provider.get_cached_api_result')
'dynamodb', account='some-account', api_kwargs={}, api_method='list_tables', region='us-east-1'
"""
resources = {}
if self.provider.cloudformation_template and template_type:
name_attribute = "{}Name".format(template_type.split('::')[-1])
for logical_id, properties in self.provider.cloudformation_template.get(
'Resources', {}).items():
if properties.get('Type') == template_type:
resource = properties.get('Properties',
{}).get(name_attribute)
if resource:
resources[resource] = re.compile(
BaseApi.RESOURCE_PATTERN.format(
re.escape(resource)), re.IGNORECASE)
api_resources = self.provider.get_cached_api_result(
service,
region=region,
account=account,
api_method=api_method,
api_kwargs=api_kwargs)[api_attribute]
if api_inner_attribute:
api_resources = (resource[api_inner_attribute]
for resource in api_resources)
if resource_converter:
api_resources = (resource_converter(resource)
for resource in api_resources)
resources.update(
(resource,
re.compile(BaseApi.RESOURCE_PATTERN.format(re.escape(resource)),
re.IGNORECASE)) for resource in api_resources)
if not resources and warn:
if not hasattr(self, '_no_resources_warnings'):
self._no_resources_warnings = set()
warning_arguments = (service, template_type, region, account)
if warning_arguments not in self._no_resources_warnings:
eprint(
"warn: no {} resources ({}) on '{}:{}', you're using this service but your AWS account and CloudFormation are empty",
*warning_arguments)
self._no_resources_warnings.add(warning_arguments)
return {}
return resources
def _reference_roles(self, permissions, config):
"""
>>> from pprint import pprint
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Args:
... pass
>>> args = Args()
>>> args.yes = True
>>> permissions = {'some': "some permissions", 'another': "another permissions"}
>>> config = {'functions': {'some': {}, 'another': {}}}
>>> ServerlessFramework("path/to/project", {}, args=args)._reference_roles(permissions, config)
>>> pprint(config)
{'functions': {'another': {'role': 'puresecAnotherRole'}, 'some': {'role': 'puresecSomeRole'}}}
>>> mock.mock(None, 'eprint')
>>> config = {'functions': {'another': {}}}
>>> ServerlessFramework("path/to/project", {}, args=args)._reference_roles(permissions, config)
>>> pprint(config)
{'functions': {'another': {'role': 'puresecAnotherRole'}}}
>>> mock.calls_for('eprint')
'warn: `{}` not found under the `functions` section in serverless.yml', 'some'
>>> config = {}
>>> ServerlessFramework("path/to/project", {}, args=args)._reference_roles(permissions, config)
>>> pprint(config)
{}
>>> mock.calls_for('eprint')
'warn: `functions` section not found in serverless.yml'
>>> args.yes = False
>>> args.reference = False
>>> args.no_reference = False
>>> args.no_input = False
>>> mock.mock(None, 'input_query', False)
>>> config = {'functions': {'some': {}, 'another': {}}}
>>> ServerlessFramework("path/to/project", {}, args=args)._reference_roles(permissions, config)
>>> pprint(config)
{'functions': {'another': {}, 'some': {}}}
>>> mock.calls_for('input_query')
'Reference functions to new roles?{}', ''
>>> mock.mock(None, 'input_query', True)
>>> config = {'functions': {'some': {}, 'another': {}}}
>>> ServerlessFramework("path/to/project", {}, args=args)._reference_roles(permissions, config)
>>> pprint(config)
{'functions': {'another': {'role': 'puresecAnotherRole'}, 'some': {'role': 'puresecSomeRole'}}}
>>> mock.calls_for('input_query')
'Reference functions to new roles?{}', ''
"""
if 'functions' not in config:
eprint("warn: `functions` section not found in serverless.yml")
return
if (self.args.yes or self.args.reference) or (
not self.args.no_reference
and not self.args.no_input and input_query(
"Reference functions to new roles?{}", self.query_suffix)):
for name in permissions.keys():
if name not in config['functions']:
eprint(
"warn: `{}` not found under the `functions` section in serverless.yml",
name)
continue
config['functions'][name]['role'] = "puresec{}Role".format(
capitalize(name))
def _get_services(self, filename, contents):
"""
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Provider:
... pass
>>> provider = Provider()
>>> provider.default_region = 'default_region'
>>> provider.default_account = 'default_account'
>>> runtime = PythonRuntime('path/to/function', resource_properties={}, provider=provider)
>>> runtime._get_services("filename.txt", ".client('s3')")
>>> pprint(normalize_dict(runtime._permissions))
{}
>>> runtime._get_services("filename.py", ".client('s3')")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", ".client('s3', region_name='localhost')")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", ".client('s3', region_name='us-east-1')")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", '''
... boto3. \\
... client('s3',
... region_name='us-east-1'
... )
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", '''
... boto3.
... client('s3',
... region_name='us-east-1', something='else'
... )
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> mock.mock(None, 'eprint')
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", '''
... boto3.
... client('s3',
... region_name=getRegion()
... )
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'*': {'default_account': {}}}}
>>> mock.calls_for('eprint')
'warn: incomprehensive region: {} (in {})', "'s3',\\n region_name=getRegion()\\n ", 'filename.py'
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", '''
... boto3.
... client('s3',
... region_name='us-' + region
... )
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'*': {'default_account': {}}}}
>>> mock.calls_for('eprint')
'warn: incomprehensive region: {} (in {})', "'s3',\\n region_name='us-' + region\\n ", 'filename.py'
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.py", '''
... boto3.
... client('s3',
... aws_access_key_id='some key'
... )
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'*': {}}}}
>>> mock.calls_for('eprint')
'warn: unknown account: {} (in {})', "'s3',\\n aws_access_key_id='some key'\\n ", 'filename.py'
"""
if not PythonRuntime.PYTHON_FILENAME_PATTERN.search(filename):
return
for service, pattern in PythonApi.SERVICE_CALL_PATTERNS:
for service_match in pattern.finditer(contents):
arguments = get_inner_parentheses(service_match.group(1))
if arguments:
# region
region = self._get_variable_from_arguments(
arguments, PythonRuntime.REGION_PATTERN)
if region is None or region == 'localhost':
region = self.provider.default_region
elif not region:
eprint("warn: incomprehensive region: {} (in {})",
arguments, filename)
region = '*'
elif not any(
pattern.match(region) for pattern in
PythonRuntime.REGION_PATTERNS.values()):
eprint("warn: incomprehensive region: {} (in {})",
arguments, filename)
region = '*'
# account
if PythonRuntime.AUTH_PATTERN.search(arguments):
eprint("warn: unknown account: {} (in {})", arguments,
filename)
account = '*'
else:
account = self.provider.default_account
else:
region = self.provider.default_region
account = self.provider.default_account
self._permissions[service][region][
account] # accessing to initialize defaultdict
def process(self):
"""
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> mock.mock(None, 'eprint')
>>> mock.mock(AwsProvider, '_process_configurations')
>>> with mock.open("path/to/cloudformation.json", 'w') as f:
... f.write('{}') and None
>>> from puresec_cli.actions.generate_roles.frameworks.base import Base as FrameworkBase
>>> class Framework(FrameworkBase):
... def get_function_name(self, name):
... return name[1:]
>>> from puresec_cli.actions.generate_roles.runtimes.aws.base import Base as RuntimeBase
>>> class RuntimeModule:
... class Runtime(RuntimeBase):
... def process(self):
... pass
>>> mock.mock(None, 'import_module', lambda name: RuntimeModule)
>>> handler = AwsProvider("path/to/project", config={}, resource_template="path/to/cloudformation.json", framework=Framework("", {}))
>>> mock.mock(AwsProvider, 'default_region', "default_region")
>>> mock.mock(AwsProvider, 'default_account', "default_account")
>>> mock.mock(handler, '_get_function_root', lambda name: "functions/{}".format(name))
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'ResourceId': {
... 'Type': 'NotLambda'
... }
... }
... })
>>> handler.process()
>>> handler._function_permissions
{}
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'ResourceId': {
... 'Type': 'AWS::Lambda::Function'
... }
... }
... })
>>> handler.process()
Traceback (most recent call last):
SystemExit: 2
>>> mock.calls_for('eprint')
'error: lambda name not specified at `{}`', 'ResourceId'
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'ResourceId': {
... 'Type': 'AWS::Lambda::Function',
... 'Properties': {
... 'FunctionName': "-functionName"
... }
... }
... }
... })
>>> handler.process()
Traceback (most recent call last):
SystemExit: 2
>>> mock.calls_for('eprint')
'error: lambda runtime not specified for `{}`', 'functionName'
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'ResourceId': {
... 'Type': 'AWS::Lambda::Function',
... 'Properties': {
... 'FunctionName': "-functionName",
... 'Runtime': "abc4.3"
... }
... }
... }
... })
>>> handler.process()
>>> mock.calls_for('eprint')
'warn: lambda runtime not yet supported: `{}` (for `{}`)', 'abc', 'functionName'
>>> handler._function_permissions
{}
>>> mock.mock(AwsProvider, 'cloudformation_template', None)
>>> handler.runtime = 'nodejs'
>>> handler.process()
>>> mock.calls_for('import_module')
'puresec_cli.actions.generate_roles.runtimes.aws.nodejs'
>>> list(handler._function_permissions.keys())
['nnamed']
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'ResourceId': {
... 'Type': 'AWS::Lambda::Function',
... 'Properties': {
... 'FunctionName': "-functionName",
... 'Runtime': "nodejs4.3"
... }
... }
... }
... })
>>> handler.process()
>>> mock.calls_for('import_module')
'puresec_cli.actions.generate_roles.runtimes.aws.nodejs'
>>> list(handler._function_permissions.keys())
['functionName']
>>> handler.function = 'functionOne'
>>> mock.mock(AwsProvider, 'cloudformation_template', {
... 'Resources': {
... 'FunctionOneId': {
... 'Type': 'AWS::Lambda::Function',
... 'Properties': {
... 'FunctionName': "-functionOne",
... 'Runtime': "nodejs4.3"
... }
... },
... 'FunctionTwoId': {
... 'Type': 'AWS::Lambda::Function',
... 'Properties': {
... 'FunctionName': "-functionTwo",
... 'Runtime': "nodejs4.3"
... }
... }
... }
... })
>>> handler.process()
>>> list(handler._function_permissions.keys())
['functionOne']
"""
self._function_real_names = {}
self._function_permissions = {}
if self.cloudformation_template:
resources = self.cloudformation_template.get('Resources', {})
else:
function_name = self.function_name or 'Unnamed'
resources = {
'{}Function'.format(camelcase(function_name)): {
'Type': 'AWS::Lambda::Function',
'Properties': {
'FunctionName': function_name,
'Runtime': self.runtime,
'Handler': self.handler,
}
}
}
for resource_id, resource_config in resources.items():
if resource_config.get('Type') == 'AWS::Lambda::Function':
# Getting name
name = resource_config.get('Properties', {}).get('FunctionName')
if not name:
eprint("error: lambda name not specified at `{}`", resource_id)
raise SystemExit(2)
if self.framework:
name = self.framework.get_function_name(name)
if self.function and self.function != name:
continue
root = os.path.join(self.path, self._get_function_root(name))
# Getting runtime
runtime = resource_config.get('Properties', {}).get('Runtime')
if not runtime:
eprint("error: lambda runtime not specified for `{}`", name)
raise SystemExit(2)
runtime = re.sub(r"[\d\.]+$", '', runtime) # ignoring runtime version (e.g nodejs4.3)
if runtime not in runtimes.__all__:
eprint("warn: lambda runtime not yet supported: `{}` (for `{}`)", runtime, name)
continue
runtime = import_module("puresec_cli.actions.generate_roles.runtimes.aws.{}".format(runtime)).Runtime(
root,
resource_properties=resource_config['Properties'],
provider=weakref.proxy(self),
)
runtime.process()
self._function_permissions[name] = runtime.permissions
self._process_configurations(name, resource_id, resource_config)
def _get_services(self, filename, contents):
"""
>>> from pprint import pprint
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> class Provider:
... pass
>>> provider = Provider()
>>> provider.default_region = 'default_region'
>>> provider.default_account = 'default_account'
>>> runtime = NodejsRuntime('path/to/function', resource_properties={}, provider=provider)
>>> runtime._get_services("filename.txt", ".S3()")
>>> pprint(normalize_dict(runtime._permissions))
{}
>>> runtime._get_services("filename.js", ".S3()")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", ".S3({ region: 'localhost' })")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", ".S3({ region: 'us-east-1' })")
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", '''
... aws.
... S3({
... region: 'us-east-1'
... })
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", '''
... aws.
... S3({
... region: 'us-east-1', something: 'else'
... })
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'us-east-1': {'default_account': {}}}}
>>> mock.mock(None, 'eprint')
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", '''
... aws.
... S3({
... region: getRegion()
... })
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'*': {'default_account': {}}}}
>>> mock.calls_for('eprint')
"warn: incomprehensive region: {} (in {}), falling back to '*'", '{\\n region: getRegion()\\n }', 'filename.js'
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", '''
... aws.
... S3({
... region: 'us-' + region
... })
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'*': {'default_account': {}}}}
>>> mock.calls_for('eprint')
"warn: incomprehensive region: {} (in {}), falling back to '*'", "{\\n region: 'us-' + region\\n }", 'filename.js'
>>> runtime._permissions.clear()
>>> runtime._get_services("filename.js", '''
... aws.
... S3({
... accessKeyId: "some key"
... })
... ''')
>>> pprint(normalize_dict(runtime._permissions))
{'s3': {'default_region': {'*': {}}}}
>>> mock.calls_for('eprint')
"warn: unknown account: {} (in {}), falling back to '*'", '{\\n accessKeyId: "some key"\\n }', 'filename.js'
"""
if not NodejsRuntime.JAVASCRIPT_FILENAME_PATTERN.search(filename):
return
for service, pattern in NodejsApi.SERVICE_CALL_PATTERNS:
for service_match in pattern.finditer(contents):
arguments = get_inner_parentheses(service_match.group(1))
if arguments:
# region
region = self._get_variable_from_arguments(
arguments, NodejsRuntime.REGION_PATTERN)
if region is None or region == 'localhost':
region = self.provider.default_region
elif not region:
eprint(
"warn: incomprehensive region: {} (in {}), falling back to '*'",
arguments, filename)
region = '*'
elif not any(
pattern.match(region) for pattern in
NodejsRuntime.REGION_PATTERNS.values()):
eprint(
"warn: incomprehensive region: {} (in {}), falling back to '*'",
arguments, filename)
region = '*'
# account
if NodejsRuntime.AUTH_PATTERN.search(arguments):
eprint(
"warn: unknown account: {} (in {}), falling back to '*'",
arguments, filename)
account = '*'
else:
account = self.provider.default_account
else:
region = self.provider.default_region
account = self.provider.default_account
self._permissions[service][region][
account] # accessing to initialize defaultdict
def _process_stream_configuration(self, name, resource_id, resource_config):
"""
>>> from tests.utils import normalize_dict
>>> from tests.mock import Mock
>>> mock = Mock(__name__)
>>> provider = AwsApi()
>>> provider.default_region = 'us-east-1'
>>> provider.default_account = '1234'
>>> mock.mock(provider, 'get_cached_api_result', {'EventSourceMappings': []})
>>> provider.cloudformation_template = {'Resources': {'Mapping': {'Type': 'AWS::Lambda::EventSourceMapping',
... 'Properties': {'FunctionName': 'SomeFunction',
... 'EventSourceArn': 'arn:aws:kinesis:us-east-1:1234:stream/SomeStream'}}}}
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> normalize_dict(provider._function_permissions)
{'functionName': {'arn:aws:kinesis:us-east-1:1234:stream/SomeStream': {'kinesis:DescribeStream', 'kinesis:GetRecords', 'kinesis:GetShardIterator', 'kinesis:ListStreams'}}}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> provider.cloudformation_template = {'Resources': {'Mapping': {'Type': 'AWS::Lambda::EventSourceMapping',
... 'Properties': {'FunctionName': 'arn:aws:lambda:us-east-1:1234:function:SomeFunction',
... 'EventSourceArn': 'arn:aws:kinesis:us-east-1:1234:stream/SomeStream'}}}}
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> normalize_dict(provider._function_permissions)
{'functionName': {'arn:aws:kinesis:us-east-1:1234:stream/SomeStream': {'kinesis:DescribeStream', 'kinesis:GetRecords', 'kinesis:GetShardIterator', 'kinesis:ListStreams'}}}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> provider.cloudformation_template = {'Resources': {'Mapping': {'Type': 'AWS::Lambda::EventSourceMapping',
... 'Properties': {'FunctionName': 'AnotherFunction',
... 'EventSourceArn': 'arn:aws:kinesis:us-east-1:1234:stream/SomeStream'}}}}
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> provider._function_permissions
{}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> provider.cloudformation_template = {'Resources': {'Mapping': {'Type': 'AWS::Lambda::EventSourceMapping',
... 'Properties': {'FunctionName': 'SomeFunction',
... 'EventSourceArn': 'arn:aws:dynamodb:us-east-1:1234:table/SomeTable'}}}}
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> provider._function_permissions
{}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> provider.cloudformation_template = {'Resources': {'Mapping': {'Type': 'AWS::DynamoDB::Table'}}}
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> provider._function_permissions
{}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> provider.cloudformation_template = None
>>> mock.mock(provider, 'get_cached_api_result', {'EventSourceMappings': [{'EventSourceArn': 'arn:aws:kinesis:us-east-1:1234:stream/SomeStream'}]})
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> normalize_dict(provider._function_permissions)
{'functionName': {'arn:aws:kinesis:us-east-1:1234:stream/SomeStream': {'kinesis:DescribeStream', 'kinesis:GetRecords', 'kinesis:GetShardIterator', 'kinesis:ListStreams'}}}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
>>> mock.mock(provider, 'get_cached_api_result', {'EventSourceMappings': [{'EventSourceArn': 'arn:aws:dynamodb:us-east-1:1234:table/SomeTable'}]})
>>> provider._function_permissions = {}
>>> provider._process_stream_configuration('functionName', 'SomeFunctionName', {'Properties': {'FunctionName': 'SomeFunction'}})
>>> provider._function_permissions
{}
>>> mock.calls_for('AwsApi.get_cached_api_result')
'lambda', account='1234', api_kwargs={'FunctionName': 'SomeFunction'}, api_method='list_event_source_mappings', region='us-east-1'
"""
function_name = resource_config['Properties']['FunctionName']
# From CloudFormation
if self.cloudformation_template:
function_id_pattern = re.compile(AwsApi.FUNCTION_ID_PATTERN.format(function_name, resource_id), re.IGNORECASE)
for other_resource_id, other_resource_config in self.cloudformation_template.get('Resources', {}).items():
if other_resource_config.get('Type') == 'AWS::Lambda::EventSourceMapping':
# either ARN, function name, or broken intrinsic function
target = other_resource_config.get('Properties', {}).get('FunctionName')
if target and function_id_pattern.search(target):
arn = other_resource_config['Properties'].get('EventSourceArn')
if not arn:
eprint("warn: event source mapping for `{}` missing `EventSourceArn`", name)
continue
service_match = AwsApi.STREAM_ARN_SERVICE_PATTERN.match(arn)
if not service_match:
continue
service = service_match.group(1)
self._function_permissions. \
setdefault(name, {}). \
setdefault(arn, set()). \
update("{}:{}".format(service, action) for action in AwsApi.STREAM_ACTIONS)
# From production environment
event_source_mappings = self.get_cached_api_result('lambda', region=self.default_region, account=self.default_account, api_method='list_event_source_mappings', api_kwargs={'FunctionName': function_name})
for event_source_mapping in event_source_mappings['EventSourceMappings']:
service_match = AwsApi.STREAM_ARN_SERVICE_PATTERN.match(event_source_mapping['EventSourceArn'])
if not service_match:
continue
service = service_match.group(1)
self._function_permissions. \
setdefault(name, {}). \
setdefault(event_source_mapping['EventSourceArn'], set()). \
update("{}:{}".format(service, action) for action in AwsApi.STREAM_ACTIONS)
