def __init__(self, dry_run=False, concurrent_workers=1, project_sema=None, max_running_operations=0): """Initialize. Args: dry_run: If True, will simply log what action would have been taken without actually applying any modifications. concurrent_workers: The number of parallel enforcement threads to execute. project_sema: An optional semaphore object, used to limit the number of concurrent projects getting written to. max_running_operations: Used to limit the number of concurrent write operations on a single project's firewall rules. Set to 0 to allow unlimited in flight asynchronous operations. """ self.enforcement_log = enforcer_log_pb2.EnforcerLog() self._dry_run = dry_run self._concurrent_workers = concurrent_workers self._project_sema = project_sema self._max_running_operations = max_running_operations self.compute = compute.ComputeClient()
def compute_client(self): """A thread local instance of compute.ComputeClient. Returns: compute.ComputeClient: A Compute API client instance. """ if not hasattr(self._local, 'compute_client'): self._local.compute_client = compute.ComputeClient( self.global_configs) return self._local.compute_client
def __init__(self, project_id, global_configs=None, compute_service=None, dry_run=False, project_sema=None, max_running_operations=0): """Initialize. Args: project_id (str): The project id for the project to enforce. global_configs (dict): Global configurations. compute_service (discovery.Resource): A Compute API service object. If not provided, one will be created using the default credentials. dry_run (bool): Set to true to ensure no actual changes are made to the project. EnforcePolicy will still return a ProjectResult proto showing the changes that would have been made. project_sema (threading.BoundedSemaphore): An optional semaphore object, used to limit the number of concurrent projects getting written to. max_running_operations (int): Used to limit the number of concurrent running operations on an API. """ self.project_id = project_id if not compute_service: gce_api = compute.ComputeClient(global_configs) compute_service = gce_api.service self.firewall_api = fe.ComputeFirewallAPI(compute_service, dry_run=dry_run) self.result = enforcer_log_pb2.ProjectResult() self.result.status = STATUS_UNSPECIFIED self.result.project_id = self.project_id self.result.timestamp_sec = datelib.Timestamp.now().AsMicroTimestamp() self._dry_run = dry_run self._project_sema = project_sema if max_running_operations: self._operation_sema = threading.BoundedSemaphore( value=max_running_operations) else: self._operation_sema = None
def enforce_firewall_policy(self, firewall_policy, compute_service=None, networks=None, allow_empty_ruleset=False, prechange_callback=None, retry_on_dry_run=False, maximum_retries=MAX_ENFORCEMENT_RETRIES): """Enforces the firewall policy on the project. Args: firewall_policy: A list of rules to enforce on the project. compute_service: A Compute service object. If not provided, one will be created using the default credentials. networks: An optional list of network names to apply the policy to. If undefined, then the policy will be applied to all networks. allow_empty_ruleset: If set to true and firewall_policy has no rules, all current firewall rules will be deleted from the project. prechange_callback: An optional callback function to pass to FirewallEnforcer.apply_firewall. It gets called if the firewall policy for a project does not match the expected policy, before any changes are actually applied. If the callback returns False then no changes will be made to the project. If it returns True then the changes will be pushed. See FirewallEnforcer.apply_firewall() docstring for more details. retry_on_dry_run: Set to True to retry applying firewall rules when the expected policy does not match the current policy when dry_run is enabled. maximum_retries: The number of times enforce_firewall_policy will attempt to set the current firewall policy to the expected firewall policy. Set to 0 to disable retry behavior. Returns: A ProjectResult proto with details on the status of the enforcement and an audit log with any changes made. """ if not compute_service: gce_api = compute.ComputeClient(self.global_configs) compute_service = gce_api.service # pylint: disable=attribute-defined-outside-init # TODO: Investigate improving to avoid the pylint disable. self.firewall_api = fe.ComputeFirewallAPI(compute_service, dry_run=self._dry_run) # pylint: disable=attribute-defined-outside-init # TODO: Investigate improving to avoid the pylint disable. self.firewall_policy = firewall_policy if networks: self.project_networks = sorted(networks) else: self.project_networks = self._get_project_networks() self.result.timestamp_sec = datelib.Timestamp.now().AsMicroTimestamp() try: # pylint: disable=attribute-defined-outside-init # TODO: Investigate improving to avoid the pylint disable. self.enforcer = self._initialize_firewall_enforcer() except EnforcementError as e: return self._set_error_status(e.reason()) except ProjectDeletedError as e: self.result.status = STATUS_DELETED self.result.status_reason = 'Project scheduled for deletion: %s' % e LOGGER.warn('Project %s scheduled for deletion: %s', self.project_id, e) return self.result except ComputeApiDisabledError as e: # Reuse the DELETED status, since the project should be moved to the # archive queue if the API is disabled. self.result.status = STATUS_DELETED self.result.status_reason = 'Project has GCE API disabled: %s' % e LOGGER.warn('Project %s has the GCE API disabled: %s', self.project_id, e) return self.result retry_enforcement_count = 0 while True: try: change_count = self.enforcer.apply_firewall( prechange_callback=prechange_callback, allow_empty_ruleset=allow_empty_ruleset, networks=self.project_networks) except fe.FirewallEnforcementFailedError as e: return self._set_error_status( 'error enforcing firewall for project: %s', e) try: # pylint: disable=attribute-defined-outside-init # TODO: Investigate improving to avoid the pylint disable. self.rules_after_enforcement = self._get_current_fw_rules() except EnforcementError as e: return self._set_error_status(e.reason()) if not change_count: # Don't attempt to retry if there were no changes. This can be # caused by the prechange callback returning false or an # exception. break if ((self._dry_run and not retry_on_dry_run) or self.rules_after_enforcement == self.expected_rules): break retry_enforcement_count += 1 if retry_enforcement_count <= maximum_retries: LOGGER.warn( 'New firewall rules do not match the expected ' 'rules enforced by the policy for project %s, ' 'retrying. (Retry #%d)', self.project_id, retry_enforcement_count) self.enforcer.refresh_current_rules() else: return self._set_error_status( 'New firewall rules do not match ' 'the expected rules enforced by ' 'the policy') self.result.status = STATUS_SUCCESS self._update_fw_results() if not self.result.gce_firewall_enforcement.rules_modified_count: LOGGER.info('Firewall policy not changed for %s', self.project_id) return self.result
def setUp(self, mock_base_client): """Set up.""" self.client = compute.ComputeClient()
def _build_pipelines(cycle_timestamp, configs, **kwargs): """Build the pipelines to load data. Args: cycle_timestamp: String of timestamp, formatted as YYYYMMDDTHHMMSSZ. configs: Dictionary of configurations. kwargs: Extra configs. Returns: List of pipelines that will be run. Raises: inventory_errors.LoadDataPipelineError. """ pipelines = [] # Commonly used clients for shared ratelimiter re-use. crm_v1_api_client = crm.CloudResourceManagerClient() dao = kwargs.get('dao') gcs_api_client = gcs.StorageClient() organization_dao_name = 'organization_dao' project_dao_name = 'project_dao' # The order here matters, e.g. groups_pipeline must come before # group_members_pipeline. pipelines = [ load_orgs_pipeline.LoadOrgsPipeline(cycle_timestamp, configs, crm_v1_api_client, kwargs.get(organization_dao_name)), load_org_iam_policies_pipeline.LoadOrgIamPoliciesPipeline( cycle_timestamp, configs, crm_v1_api_client, kwargs.get(organization_dao_name)), load_projects_pipeline.LoadProjectsPipeline( cycle_timestamp, configs, crm_v1_api_client, kwargs.get(project_dao_name)), load_projects_iam_policies_pipeline.LoadProjectsIamPoliciesPipeline( cycle_timestamp, configs, crm_v1_api_client, kwargs.get(project_dao_name)), load_projects_buckets_pipeline.LoadProjectsBucketsPipeline( cycle_timestamp, configs, gcs_api_client, kwargs.get(project_dao_name)), load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline( cycle_timestamp, configs, gcs_api_client, kwargs.get('bucket_dao')), load_projects_cloudsql_pipeline.LoadProjectsCloudsqlPipeline( cycle_timestamp, configs, cloudsql.CloudsqlClient(), kwargs.get('cloudsql_dao')), load_forwarding_rules_pipeline.LoadForwardingRulesPipeline( cycle_timestamp, configs, compute.ComputeClient(), kwargs.get('fwd_rules_dao')), load_folders_pipeline.LoadFoldersPipeline( cycle_timestamp, configs, crm.CloudResourceManagerClient(version='v2beta1'), dao), load_bigquery_datasets_pipeline.LoadBigQueryDatasetsPipeline( cycle_timestamp, configs, bq.BigQueryClient(), dao), load_firewall_rules_pipeline.LoadFirewallRulesPipeline( cycle_timestamp, configs, compute.ComputeClient(version='beta'), kwargs.get(project_dao_name)), ] if configs.get('inventory_groups'): if util.can_inventory_groups(configs): admin_api_client = ad.AdminDirectoryClient() pipelines.extend([ load_groups_pipeline.LoadGroupsPipeline( cycle_timestamp, configs, admin_api_client, dao), load_group_members_pipeline.LoadGroupMembersPipeline( cycle_timestamp, configs, admin_api_client, dao) ]) else: raise inventory_errors.LoadDataPipelineError( 'Unable to inventory groups with specified arguments:\n%s', configs) return pipelines
def test_no_quota(self, mock_google_credential): """Verify no rate limiter is used if the configuration is missing.""" gce_api_client = compute.ComputeClient(global_configs={}) self.assertEqual(None, gce_api_client.repository._rate_limiter)
def setUpClass(cls, mock_google_credential): """Set up.""" fake_global_configs = {'max_compute_api_calls_per_second': 2000} cls.gce_api_client = compute.ComputeClient( global_configs=fake_global_configs) cls.project_id = fake_compute.FAKE_PROJECT_ID
def setUp(self, mock_google_credential, mock_discovery): """Set up.""" fake_global_configs = {'max_compute_api_calls_per_second': 20} self.client = compute.ComputeClient(global_configs=fake_global_configs)