def test_condition(self): registry = PluginRegistry('dummy') @registry.register('mud', condition=False) class _plugin_impl: pass self.assertEqual(list(registry.keys()), []) def _plugin_impl_func(): pass registry.register('concrete', _plugin_impl_func, condition=False) self.assertEqual(list(registry.keys()), [])
def test_unregister(self): registry = PluginRegistry('dummy') klass = lambda: 1 # NOQA registry.register('dust', klass) self.assertEqual(list(registry.keys()), ['dust']) self.assertEqual(list(registry.values()), [klass]) registry.unregister('dust')
def test_empty_with_lazy_load(self): empty_registry = PluginRegistry('stuff') self.patch(schema, 'clouds', empty_registry) policy_schema = generate() self.assertEqual( policy_schema['properties']['policies']['items'], {'type': 'object'})
def test_registry_getitem_keyerror(self): registry = PluginRegistry('dummy') try: registry['xyz'] except KeyError: pass else: self.fail('should have raised keyerror')
def test_event_subscriber(self): observed = [] def observer(*args): observed.append(args) registry = PluginRegistry('dummy') registry.subscribe(PluginRegistry.EVENT_REGISTER, observer) @registry.register('water') class _plugin_impl: pass self.assertEqual(observed[0], (registry, _plugin_impl)) self.assertEqual(list(registry.keys()), ['water']) self.assertRaises(ValueError, registry.subscribe, 'foo', observer)
def test_event_subscriber(self): observed = [] def observer(*args): observed.append(args) registry = PluginRegistry('dummy') @registry.register('hot') class _plugin_impl1: pass registry.subscribe(observer) @registry.register('water') class _plugin_impl2: pass self.assertEqual(observed, []) registry.notify(_plugin_impl1) registry.notify(_plugin_impl2) self.assertEqual(observed[1], (registry, _plugin_impl2)) self.assertEqual(list(sorted(registry.keys())), ['hot', 'water'])
class Azure(Provider): display_name = 'Azure' resource_prefix = 'azure' resources = PluginRegistry('%s.resources' % resource_prefix) resource_map = ResourceMap region_to_cloud = { 'AzureCloud': AZURE_PUBLIC_CLOUD, 'AzureChinaCloud': AZURE_CHINA_CLOUD, 'AzureGermanCloud': AZURE_GERMAN_CLOUD, 'AzureUSGovernment': AZURE_US_GOV_CLOUD } cloud_endpoints = None def initialize(self, options): self.cloud_endpoints = self._get_cloud_endpoints(options) options['region'] = self.cloud_endpoints.name if options['account_id'] is None: session = local_session(self.get_session_factory(options)) options['account_id'] = session.get_subscription_id() options['cache'] = 'memory' return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): return partial(Session, subscription_id=options.account_id, authorization_file=options.authorization_file, cloud_endpoints=self.cloud_endpoints) def _get_cloud_endpoints(self, options): cloud_list = options.get('regions') if not cloud_list: return AZURE_PUBLIC_CLOUD elif len(cloud_list) > 1: log.error('Multiple Azure Clouds provided. Please pass in only one.') sys.exit(1) # Only support passing in one cloud at a time cloud = self.region_to_cloud.get(cloud_list[0]) if cloud: return cloud else: log.error('Region Flag: %s not recognized. Available values: %s.', cloud_list[0], ", ".join(self.region_to_cloud.keys())) sys.exit(1)
class Azure(Provider): resource_prefix = 'azure' resources = PluginRegistry('%s.resources' % resource_prefix) def initialize(self, options): return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): return Session
class Kubernetes(Provider): resource_prefix = 'k8s' resources = PluginRegistry('%s.resources' % resource_prefix) def initialize(self, options): return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): """Get a credential/session factory for api usage.""" return Session
class GoogleCloud(Provider): resource_prefix = 'gcp' resources = PluginRegistry('%s.resources' % resource_prefix) def initialize(self, options): return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): """Get a credential/session factory for api usage.""" return partial(Session, project_id=options.account_id)
class Azure(Provider): resource_prefix = 'azure' resources = PluginRegistry('%s.resources' % resource_prefix) def initialize(self, options): return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): return partial(Session, subscription_id=options.account_id, authorization_file=options.authorization_file)
class CustodianProvider(Provider): display_name = "Custodian Core" resources = PluginRegistry("policy") resource_prefix = "c7n" # lazy load chicken sacrifice resource_map = {"c7n.data": "c7n.data.Data"} def get_session_factory(self, config): return NullSession() def initialize(self, options): return def initialize_policies(self, policy_collection, options): return policy_collection
class OpenStack(Provider): display_name = 'openstack' resource_prefix = 'openstack' resources = PluginRegistry('%s.resources' % resource_prefix) resource_map = ResourceMap def initialize(self, options): return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): """Get a credential/session factory for api usage.""" return Session
class Azure(Provider): display_name = 'Azure' resource_prefix = 'azure' resources = PluginRegistry('%s.resources' % resource_prefix) def initialize(self, options): if options['account_id'] is None: session = local_session(self.get_session_factory(options)) options['account_id'] = session.get_subscription_id() return options def initialize_policies(self, policy_collection, options): return policy_collection def get_session_factory(self, options): return partial(Session, subscription_id=options.account_id, authorization_file=options.authorization_file)
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, division, print_function, unicode_literals import abc import six import importlib from c7n.registry import PluginRegistry clouds = PluginRegistry('c7n.providers') @six.add_metaclass(abc.ABCMeta) class Provider(object): """Provider Base Class""" @abc.abstractproperty def display_name(self): """display name for the provider in docs""" @abc.abstractproperty def resources(self): """resources registry for this cloud provider""" @abc.abstractproperty def resource_prefix(self):
class ServerlessExecutionMode(PolicyExecutionMode): def run(self, event=None, lambda_context=None): """Run the actual policy.""" raise NotImplementedError("subclass responsibility") def get_logs(self, start, end): """Retrieve logs for the policy""" raise NotImplementedError("subclass responsibility") def provision(self): """Provision any resources needed for the policy.""" raise NotImplementedError("subclass responsibility") execution = PluginRegistry('c7n.execution') @execution.register('pull') class PullMode(PolicyExecutionMode): """Pull mode execution of a policy. Queries resources from cloud provider for filtering and actions. """ schema = utils.type_schema('pull') def run(self, *args, **kw): if not self.is_runnable(): return
class AWS(Provider): display_name = 'AWS' resource_prefix = 'aws' # legacy path for older plugins resources = PluginRegistry('resources') # import paths for resources resource_map = ResourceMap def initialize(self, options): """ """ _default_region(options) _default_account_id(options) if options.tracer and options.tracer.startswith('xray') and HAVE_XRAY: XrayTracer.initialize() return options def get_session_factory(self, options): return SessionFactory(options.region, options.profile, options.assume_role, options.external_id) def initialize_policies(self, policy_collection, options): """Return a set of policies targetted to the given regions. Supports symbolic regions like 'all'. This will automatically filter out policies if their being targetted to a region that does not support the service. Global services will target a single region (us-east-1 if only all specified, else first region in the list). Note for region partitions (govcloud and china) an explicit region from the partition must be passed in. """ from c7n.policy import Policy, PolicyCollection policies = [] service_region_map, resource_service_map = get_service_region_map( options.regions, policy_collection.resource_types) if 'all' in options.regions: enabled_regions = set([ r['RegionName'] for r in get_profile_session(options).client( 'ec2').describe_regions( Filters=[{ 'Name': 'opt-in-status', 'Values': ['opt-in-not-required', 'opted-in'] }]).get('Regions') ]) for p in policy_collection: if 'aws.' in p.resource_type: _, resource_type = p.resource_type.split('.', 1) else: resource_type = p.resource_type available_regions = service_region_map.get( resource_service_map.get(resource_type), ()) # its a global service/endpoint, use user provided region # or us-east-1. if not available_regions and options.regions: candidates = [r for r in options.regions if r != 'all'] candidate = candidates and candidates[0] or 'us-east-1' svc_regions = [candidate] elif 'all' in options.regions: svc_regions = list( set(available_regions).intersection(enabled_regions)) else: svc_regions = options.regions for region in svc_regions: if available_regions and region not in available_regions: level = ('all' in options.regions and logging.DEBUG or logging.WARNING) # TODO: fixme policy_collection.log.log( level, "policy:%s resources:%s not available in region:%s", p.name, p.resource_type, region) continue options_copy = copy.copy(options) options_copy.region = str(region) if len(options.regions ) > 1 or 'all' in options.regions and getattr( options, 'output_dir', None): options_copy.output_dir = join_output( options.output_dir, region) policies.append( Policy( p.data, options_copy, session_factory=policy_collection.session_factory())) return PolicyCollection( # order policies by region to minimize local session invalidation. # note relative ordering of policies must be preserved, python sort # is stable. sorted(policies, key=operator.attrgetter('options.region')), options)
def test_unregister(self): registry = PluginRegistry('dummy') registry.register('dust', klass=lambda: 1) self.assertEqual(list(registry.keys()), ['dust']) registry.unregister('dust')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import deque import logging from c7n import cache from c7n.executor import ThreadPoolExecutor from c7n.provider import clouds from c7n.registry import PluginRegistry from c7n.resources import load_resources try: from c7n.resources.aws import AWS resources = AWS.resources except ImportError: resources = PluginRegistry('resources') from c7n.utils import dumps def iter_filters(filters, block_end=False): queue = deque(filters) while queue: f = queue.popleft() if f is not None and f.type in ('or', 'and', 'not'): if block_end: queue.appendleft(None) for gf in f.filters: queue.appendleft(gf) yield f
class AWS(object): resource_prefix = 'aws' # legacy path for older plugins resources = PluginRegistry('resources')
class GoogleCloud(object): resource_prefix = 'gcp' resources = PluginRegistry('%s.resources' % resource_prefix)
}, 'regions': { 'type': 'array', 'items': { 'type': 'string' } } } } } } } retry = get_retry(('Throttling', ), log_retries=True) indexers = PluginRegistry('policy-metrics-indexers') class Indexer: """ Metrics indexer """ def get_indexer(config, **kwargs): itype = config['indexer']['type'] klass = indexers.get(itype) return klass(config, **kwargs) @indexers.register('es') class ElasticSearchIndexer(Indexer):
class AwsCloudControl(AWS): display_name = "AWS Cloud Control" resource_prefix = ("awscc",) resources = PluginRegistry("%s.resources" % resource_prefix) resource_map = get_resource_map()
class AWS(object): resource_prefix = 'aws' # legacy path for older plugins resources = PluginRegistry('resources') def initialize(self, options): """ """ _default_region(options) _default_account_id(options) return options def get_session_factory(self, options): return SessionFactory(options.region, options.profile, options.assume_role, options.external_id) def initialize_policies(self, policy_collection, options): """Return a set of policies targetted to the given regions. Supports symbolic regions like 'all'. This will automatically filter out policies if their being targetted to a region that does not support the service. Global services will target a single region (us-east-1 if only all specified, else first region in the list). Note for region partitions (govcloud and china) an explicit region from the partition must be passed in. """ from c7n.policy import Policy, PolicyCollection policies = [] service_region_map, resource_service_map = get_service_region_map( options.regions, policy_collection.resource_types) for p in policy_collection: if 'aws.' in p.resource_type: _, resource_type = p.resource_type.split('.', 1) else: resource_type = p.resource_type available_regions = service_region_map.get( resource_service_map.get(resource_type), ()) # its a global service/endpoint, use user provided region # or us-east-1. if not available_regions and options.regions: candidates = [r for r in options.regions if r != 'all'] candidate = candidates and candidates[0] or 'us-east-1' svc_regions = [candidate] elif 'all' in options.regions: svc_regions = available_regions else: svc_regions = options.regions for region in svc_regions: if available_regions and region not in available_regions: level = ('all' in options.regions and logging.DEBUG or logging.WARNING) # TODO: fixme policy_collection.log.log( level, "policy:%s resources:%s not available in region:%s", p.name, p.resource_type, region) continue options_copy = copy.copy(options) options_copy.region = str(region) if len(options.regions ) > 1 or 'all' in options.regions and getattr( options, 'output_dir', None): options_copy.output_dir = (options.output_dir.rstrip('/') + '/%s' % region) policies.append( Policy( p.data, options_copy, session_factory=policy_collection.session_factory())) return PolicyCollection( # order policies by region to minimize local session invalidation. # note relative ordering of policies must be preserved, python sort # is stable. sorted(policies, key=operator.attrgetter('options.region')), options)
import shutil import tempfile import os from boto3.s3.transfer import S3Transfer from c7n.registry import PluginRegistry from c7n.log import CloudWatchLogHandler from c7n.utils import local_session, parse_s3, get_retry DEFAULT_NAMESPACE = "CloudMaid" log = logging.getLogger('custodian.output') blob_outputs = PluginRegistry('c7n.blob-outputs') class MetricsOutput(object): """Send metrics data to cloudwatch """ permissions = ("cloudWatch:PutMetricData", ) retry = staticmethod(get_retry(('Throttling', ))) @staticmethod def select(metrics_enabled): if metrics_enabled: return MetricsOutput return NullMetricsOutput
class Azure(object): resource_prefix = 'azure' resources = PluginRegistry('%s.resources' % resource_prefix)