def test_condition(self):
registry = PluginRegistry('dummy')
@registry.register('mud', condition=False)
class _plugin_impl:
pass
self.assertEqual(list(registry.keys()), [])
def _plugin_impl_func():
pass
registry.register('concrete', _plugin_impl_func, condition=False)
self.assertEqual(list(registry.keys()), [])
def test_unregister(self):
registry = PluginRegistry('dummy')
klass = lambda: 1 # NOQA
registry.register('dust', klass)
self.assertEqual(list(registry.keys()), ['dust'])
self.assertEqual(list(registry.values()), [klass])
registry.unregister('dust')
def test_empty_with_lazy_load(self):
empty_registry = PluginRegistry('stuff')
self.patch(schema, 'clouds', empty_registry)
policy_schema = generate()
self.assertEqual(
policy_schema['properties']['policies']['items'],
{'type': 'object'})
def test_registry_getitem_keyerror(self):
registry = PluginRegistry('dummy')
try:
registry['xyz']
except KeyError:
pass
else:
self.fail('should have raised keyerror')
def test_event_subscriber(self):
observed = []
def observer(*args):
observed.append(args)
registry = PluginRegistry('dummy')
registry.subscribe(PluginRegistry.EVENT_REGISTER, observer)
@registry.register('water')
class _plugin_impl:
pass
self.assertEqual(observed[0], (registry, _plugin_impl))
self.assertEqual(list(registry.keys()), ['water'])
self.assertRaises(ValueError, registry.subscribe, 'foo', observer)
def test_event_subscriber(self):
observed = []
def observer(*args):
observed.append(args)
registry = PluginRegistry('dummy')
registry.subscribe(PluginRegistry.EVENT_REGISTER, observer)
@registry.register('water')
class _plugin_impl:
pass
self.assertEqual(observed[0], (registry, _plugin_impl))
self.assertEqual(list(registry.keys()), ['water'])
self.assertRaises(ValueError, registry.subscribe, 'foo', observer)
def test_event_subscriber(self):
observed = []
def observer(*args):
observed.append(args)
registry = PluginRegistry('dummy')
@registry.register('hot')
class _plugin_impl1:
pass
registry.subscribe(observer)
@registry.register('water')
class _plugin_impl2:
pass
self.assertEqual(observed, [])
registry.notify(_plugin_impl1)
registry.notify(_plugin_impl2)
self.assertEqual(observed[1], (registry, _plugin_impl2))
self.assertEqual(list(sorted(registry.keys())), ['hot', 'water'])
class Azure(Provider):
display_name = 'Azure'
resource_prefix = 'azure'
resources = PluginRegistry('%s.resources' % resource_prefix)
resource_map = ResourceMap
region_to_cloud = {
'AzureCloud': AZURE_PUBLIC_CLOUD,
'AzureChinaCloud': AZURE_CHINA_CLOUD,
'AzureGermanCloud': AZURE_GERMAN_CLOUD,
'AzureUSGovernment': AZURE_US_GOV_CLOUD
}
cloud_endpoints = None
def initialize(self, options):
self.cloud_endpoints = self._get_cloud_endpoints(options)
options['region'] = self.cloud_endpoints.name
if options['account_id'] is None:
session = local_session(self.get_session_factory(options))
options['account_id'] = session.get_subscription_id()
options['cache'] = 'memory'
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
return partial(Session,
subscription_id=options.account_id,
authorization_file=options.authorization_file,
cloud_endpoints=self.cloud_endpoints)
def _get_cloud_endpoints(self, options):
cloud_list = options.get('regions')
if not cloud_list:
return AZURE_PUBLIC_CLOUD
elif len(cloud_list) > 1:
log.error('Multiple Azure Clouds provided. Please pass in only one.')
sys.exit(1)
# Only support passing in one cloud at a time
cloud = self.region_to_cloud.get(cloud_list[0])
if cloud:
return cloud
else:
log.error('Region Flag: %s not recognized. Available values: %s.',
cloud_list[0], ", ".join(self.region_to_cloud.keys()))
sys.exit(1)
class Azure(Provider):
resource_prefix = 'azure'
resources = PluginRegistry('%s.resources' % resource_prefix)
def initialize(self, options):
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
return Session
class Kubernetes(Provider):
resource_prefix = 'k8s'
resources = PluginRegistry('%s.resources' % resource_prefix)
def initialize(self, options):
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
"""Get a credential/session factory for api usage."""
return Session
class GoogleCloud(Provider):
resource_prefix = 'gcp'
resources = PluginRegistry('%s.resources' % resource_prefix)
def initialize(self, options):
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
"""Get a credential/session factory for api usage."""
return partial(Session, project_id=options.account_id)
class Azure(Provider):
resource_prefix = 'azure'
resources = PluginRegistry('%s.resources' % resource_prefix)
def initialize(self, options):
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
return partial(Session,
subscription_id=options.account_id,
authorization_file=options.authorization_file)
class CustodianProvider(Provider):
display_name = "Custodian Core"
resources = PluginRegistry("policy")
resource_prefix = "c7n"
# lazy load chicken sacrifice
resource_map = {"c7n.data": "c7n.data.Data"}
def get_session_factory(self, config):
return NullSession()
def initialize(self, options):
return
def initialize_policies(self, policy_collection, options):
return policy_collection
class OpenStack(Provider):
display_name = 'openstack'
resource_prefix = 'openstack'
resources = PluginRegistry('%s.resources' % resource_prefix)
resource_map = ResourceMap
def initialize(self, options):
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
"""Get a credential/session factory for api usage."""
return Session
class Azure(Provider):
display_name = 'Azure'
resource_prefix = 'azure'
resources = PluginRegistry('%s.resources' % resource_prefix)
def initialize(self, options):
if options['account_id'] is None:
session = local_session(self.get_session_factory(options))
options['account_id'] = session.get_subscription_id()
return options
def initialize_policies(self, policy_collection, options):
return policy_collection
def get_session_factory(self, options):
return partial(Session,
subscription_id=options.account_id,
authorization_file=options.authorization_file)
def test_condition(self):
registry = PluginRegistry('dummy')
@registry.register('mud', condition=False)
class _plugin_impl:
pass
self.assertEqual(list(registry.keys()), [])
def _plugin_impl_func():
pass
registry.register('concrete', _plugin_impl_func, condition=False)
self.assertEqual(list(registry.keys()), [])
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals
import abc
import six
import importlib
from c7n.registry import PluginRegistry
clouds = PluginRegistry('c7n.providers')
@six.add_metaclass(abc.ABCMeta)
class Provider(object):
"""Provider Base Class"""
@abc.abstractproperty
def display_name(self):
"""display name for the provider in docs"""
@abc.abstractproperty
def resources(self):
"""resources registry for this cloud provider"""
@abc.abstractproperty
def resource_prefix(self):
class ServerlessExecutionMode(PolicyExecutionMode):
def run(self, event=None, lambda_context=None):
"""Run the actual policy."""
raise NotImplementedError("subclass responsibility")
def get_logs(self, start, end):
"""Retrieve logs for the policy"""
raise NotImplementedError("subclass responsibility")
def provision(self):
"""Provision any resources needed for the policy."""
raise NotImplementedError("subclass responsibility")
execution = PluginRegistry('c7n.execution')
@execution.register('pull')
class PullMode(PolicyExecutionMode):
"""Pull mode execution of a policy.
Queries resources from cloud provider for filtering and actions.
"""
schema = utils.type_schema('pull')
def run(self, *args, **kw):
if not self.is_runnable():
return
class AWS(Provider):
display_name = 'AWS'
resource_prefix = 'aws'
# legacy path for older plugins
resources = PluginRegistry('resources')
# import paths for resources
resource_map = ResourceMap
def initialize(self, options):
"""
"""
_default_region(options)
_default_account_id(options)
if options.tracer and options.tracer.startswith('xray') and HAVE_XRAY:
XrayTracer.initialize()
return options
def get_session_factory(self, options):
return SessionFactory(options.region, options.profile,
options.assume_role, options.external_id)
def initialize_policies(self, policy_collection, options):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically
filter out policies if their being targetted to a region that
does not support the service. Global services will target a
single region (us-east-1 if only all specified, else first
region in the list).
Note for region partitions (govcloud and china) an explicit
region from the partition must be passed in.
"""
from c7n.policy import Policy, PolicyCollection
policies = []
service_region_map, resource_service_map = get_service_region_map(
options.regions, policy_collection.resource_types)
if 'all' in options.regions:
enabled_regions = set([
r['RegionName'] for r in get_profile_session(options).client(
'ec2').describe_regions(
Filters=[{
'Name': 'opt-in-status',
'Values': ['opt-in-not-required', 'opted-in']
}]).get('Regions')
])
for p in policy_collection:
if 'aws.' in p.resource_type:
_, resource_type = p.resource_type.split('.', 1)
else:
resource_type = p.resource_type
available_regions = service_region_map.get(
resource_service_map.get(resource_type), ())
# its a global service/endpoint, use user provided region
# or us-east-1.
if not available_regions and options.regions:
candidates = [r for r in options.regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in options.regions:
svc_regions = list(
set(available_regions).intersection(enabled_regions))
else:
svc_regions = options.regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = ('all' in options.regions and logging.DEBUG
or logging.WARNING)
# TODO: fixme
policy_collection.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(options)
options_copy.region = str(region)
if len(options.regions
) > 1 or 'all' in options.regions and getattr(
options, 'output_dir', None):
options_copy.output_dir = join_output(
options.output_dir, region)
policies.append(
Policy(
p.data,
options_copy,
session_factory=policy_collection.session_factory()))
return PolicyCollection(
# order policies by region to minimize local session invalidation.
# note relative ordering of policies must be preserved, python sort
# is stable.
sorted(policies, key=operator.attrgetter('options.region')),
options)
def test_unregister(self):
registry = PluginRegistry('dummy')
registry.register('dust', klass=lambda: 1)
self.assertEqual(list(registry.keys()), ['dust'])
registry.unregister('dust')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import deque
import logging
from c7n import cache
from c7n.executor import ThreadPoolExecutor
from c7n.provider import clouds
from c7n.registry import PluginRegistry
from c7n.resources import load_resources
try:
from c7n.resources.aws import AWS
resources = AWS.resources
except ImportError:
resources = PluginRegistry('resources')
from c7n.utils import dumps
def iter_filters(filters, block_end=False):
queue = deque(filters)
while queue:
f = queue.popleft()
if f is not None and f.type in ('or', 'and', 'not'):
if block_end:
queue.appendleft(None)
for gf in f.filters:
queue.appendleft(gf)
yield f
class AWS(object):
resource_prefix = 'aws'
# legacy path for older plugins
resources = PluginRegistry('resources')
class GoogleCloud(object):
resource_prefix = 'gcp'
resources = PluginRegistry('%s.resources' % resource_prefix)
},
'regions': {
'type': 'array',
'items': {
'type': 'string'
}
}
}
}
}
}
}
retry = get_retry(('Throttling', ), log_retries=True)
indexers = PluginRegistry('policy-metrics-indexers')
class Indexer:
""" Metrics indexer
"""
def get_indexer(config, **kwargs):
itype = config['indexer']['type']
klass = indexers.get(itype)
return klass(config, **kwargs)
@indexers.register('es')
class ElasticSearchIndexer(Indexer):
class AwsCloudControl(AWS):
display_name = "AWS Cloud Control"
resource_prefix = ("awscc",)
resources = PluginRegistry("%s.resources" % resource_prefix)
resource_map = get_resource_map()
class AWS(object):
resource_prefix = 'aws'
# legacy path for older plugins
resources = PluginRegistry('resources')
def initialize(self, options):
"""
"""
_default_region(options)
_default_account_id(options)
return options
def get_session_factory(self, options):
return SessionFactory(options.region, options.profile,
options.assume_role, options.external_id)
def initialize_policies(self, policy_collection, options):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically
filter out policies if their being targetted to a region that
does not support the service. Global services will target a
single region (us-east-1 if only all specified, else first
region in the list).
Note for region partitions (govcloud and china) an explicit
region from the partition must be passed in.
"""
from c7n.policy import Policy, PolicyCollection
policies = []
service_region_map, resource_service_map = get_service_region_map(
options.regions, policy_collection.resource_types)
for p in policy_collection:
if 'aws.' in p.resource_type:
_, resource_type = p.resource_type.split('.', 1)
else:
resource_type = p.resource_type
available_regions = service_region_map.get(
resource_service_map.get(resource_type), ())
# its a global service/endpoint, use user provided region
# or us-east-1.
if not available_regions and options.regions:
candidates = [r for r in options.regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in options.regions:
svc_regions = available_regions
else:
svc_regions = options.regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = ('all' in options.regions and logging.DEBUG
or logging.WARNING)
# TODO: fixme
policy_collection.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(options)
options_copy.region = str(region)
if len(options.regions
) > 1 or 'all' in options.regions and getattr(
options, 'output_dir', None):
options_copy.output_dir = (options.output_dir.rstrip('/') +
'/%s' % region)
policies.append(
Policy(
p.data,
options_copy,
session_factory=policy_collection.session_factory()))
return PolicyCollection(
# order policies by region to minimize local session invalidation.
# note relative ordering of policies must be preserved, python sort
# is stable.
sorted(policies, key=operator.attrgetter('options.region')),
options)
import shutil
import tempfile
import os
from boto3.s3.transfer import S3Transfer
from c7n.registry import PluginRegistry
from c7n.log import CloudWatchLogHandler
from c7n.utils import local_session, parse_s3, get_retry
DEFAULT_NAMESPACE = "CloudMaid"
log = logging.getLogger('custodian.output')
blob_outputs = PluginRegistry('c7n.blob-outputs')
class MetricsOutput(object):
"""Send metrics data to cloudwatch
"""
permissions = ("cloudWatch:PutMetricData", )
retry = staticmethod(get_retry(('Throttling', )))
@staticmethod
def select(metrics_enabled):
if metrics_enabled:
return MetricsOutput
return NullMetricsOutput
class Azure(object):
resource_prefix = 'azure'
resources = PluginRegistry('%s.resources' % resource_prefix)
