def __init__(self,
dry_run=False,
concurrent_workers=1,
project_sema=None,
max_running_operations=0):
"""Initialize.
Args:
dry_run: If True, will simply log what action would have been taken
without actually applying any modifications.
concurrent_workers: The number of parallel enforcement threads to
execute.
project_sema: An optional semaphore object, used to limit the number
of concurrent projects getting written to.
max_running_operations: Used to limit the number of concurrent write
operations on a single project's firewall rules. Set to 0 to
allow unlimited in flight asynchronous operations.
"""
self.enforcement_log = enforcer_log_pb2.EnforcerLog()
self._dry_run = dry_run
self._concurrent_workers = concurrent_workers
self._project_sema = project_sema
self._max_running_operations = max_running_operations
self.compute = compute.ComputeClient()
def compute_client(self):
"""A thread local instance of compute.ComputeClient.
Returns:
compute.ComputeClient: A Compute API client instance.
"""
if not hasattr(self._local, 'compute_client'):
self._local.compute_client = compute.ComputeClient(
self.global_configs)
return self._local.compute_client
def __init__(self,
project_id,
global_configs=None,
compute_service=None,
dry_run=False,
project_sema=None,
max_running_operations=0):
"""Initialize.
Args:
project_id (str): The project id for the project to enforce.
global_configs (dict): Global configurations.
compute_service (discovery.Resource): A Compute API service object.
If not provided, one will be created using the default
credentials.
dry_run (bool): Set to true to ensure no actual changes are made to
the project. EnforcePolicy will still return a ProjectResult
proto showing the changes that would have been made.
project_sema (threading.BoundedSemaphore): An optional semaphore
object, used to limit the number of concurrent projects getting
written to.
max_running_operations (int): Used to limit the number of concurrent
running operations on an API.
"""
self.project_id = project_id
if not compute_service:
gce_api = compute.ComputeClient(global_configs)
compute_service = gce_api.service
self.firewall_api = fe.ComputeFirewallAPI(compute_service,
dry_run=dry_run)
self.result = enforcer_log_pb2.ProjectResult()
self.result.status = STATUS_UNSPECIFIED
self.result.project_id = self.project_id
self.result.timestamp_sec = datelib.Timestamp.now().AsMicroTimestamp()
self._dry_run = dry_run
self._project_sema = project_sema
if max_running_operations:
self._operation_sema = threading.BoundedSemaphore(
value=max_running_operations)
else:
self._operation_sema = None
def enforce_firewall_policy(self,
firewall_policy,
compute_service=None,
networks=None,
allow_empty_ruleset=False,
prechange_callback=None,
retry_on_dry_run=False,
maximum_retries=MAX_ENFORCEMENT_RETRIES):
"""Enforces the firewall policy on the project.
Args:
firewall_policy: A list of rules to enforce on the project.
compute_service: A Compute service object. If not provided, one will
be created using the default credentials.
networks: An optional list of network names to apply the policy to. If
undefined, then the policy will be applied to all networks.
allow_empty_ruleset: If set to true and firewall_policy has no rules,
all current firewall rules will be deleted from the project.
prechange_callback: An optional callback function to pass to
FirewallEnforcer.apply_firewall. It gets called if the
firewall policy for a project does not match the expected policy,
before any changes are actually applied. If the callback returns
False then no changes will be made to the project. If it returns
True then the changes will be pushed.
See FirewallEnforcer.apply_firewall() docstring for more details.
retry_on_dry_run: Set to True to retry applying firewall rules when
the expected policy does not match the current policy when
dry_run is enabled.
maximum_retries: The number of times enforce_firewall_policy will
attempt to set the current firewall policy to the expected
firewall policy. Set to 0 to disable retry behavior.
Returns:
A ProjectResult proto with details on the status of the enforcement
and an audit log with any changes made.
"""
if not compute_service:
gce_api = compute.ComputeClient(self.global_configs)
compute_service = gce_api.service
# pylint: disable=attribute-defined-outside-init
# TODO: Investigate improving to avoid the pylint disable.
self.firewall_api = fe.ComputeFirewallAPI(compute_service,
dry_run=self._dry_run)
# pylint: disable=attribute-defined-outside-init
# TODO: Investigate improving to avoid the pylint disable.
self.firewall_policy = firewall_policy
if networks:
self.project_networks = sorted(networks)
else:
self.project_networks = self._get_project_networks()
self.result.timestamp_sec = datelib.Timestamp.now().AsMicroTimestamp()
try:
# pylint: disable=attribute-defined-outside-init
# TODO: Investigate improving to avoid the pylint disable.
self.enforcer = self._initialize_firewall_enforcer()
except EnforcementError as e:
return self._set_error_status(e.reason())
except ProjectDeletedError as e:
self.result.status = STATUS_DELETED
self.result.status_reason = 'Project scheduled for deletion: %s' % e
LOGGER.warn('Project %s scheduled for deletion: %s',
self.project_id, e)
return self.result
except ComputeApiDisabledError as e:
# Reuse the DELETED status, since the project should be moved to the
# archive queue if the API is disabled.
self.result.status = STATUS_DELETED
self.result.status_reason = 'Project has GCE API disabled: %s' % e
LOGGER.warn('Project %s has the GCE API disabled: %s',
self.project_id, e)
return self.result
retry_enforcement_count = 0
while True:
try:
change_count = self.enforcer.apply_firewall(
prechange_callback=prechange_callback,
allow_empty_ruleset=allow_empty_ruleset,
networks=self.project_networks)
except fe.FirewallEnforcementFailedError as e:
return self._set_error_status(
'error enforcing firewall for project: %s', e)
try:
# pylint: disable=attribute-defined-outside-init
# TODO: Investigate improving to avoid the pylint disable.
self.rules_after_enforcement = self._get_current_fw_rules()
except EnforcementError as e:
return self._set_error_status(e.reason())
if not change_count:
# Don't attempt to retry if there were no changes. This can be
# caused by the prechange callback returning false or an
# exception.
break
if ((self._dry_run and not retry_on_dry_run)
or self.rules_after_enforcement == self.expected_rules):
break
retry_enforcement_count += 1
if retry_enforcement_count <= maximum_retries:
LOGGER.warn(
'New firewall rules do not match the expected '
'rules enforced by the policy for project %s, '
'retrying. (Retry #%d)', self.project_id,
retry_enforcement_count)
self.enforcer.refresh_current_rules()
else:
return self._set_error_status(
'New firewall rules do not match '
'the expected rules enforced by '
'the policy')
self.result.status = STATUS_SUCCESS
self._update_fw_results()
if not self.result.gce_firewall_enforcement.rules_modified_count:
LOGGER.info('Firewall policy not changed for %s', self.project_id)
return self.result
def setUp(self, mock_base_client):
"""Set up."""
self.client = compute.ComputeClient()
def _build_pipelines(cycle_timestamp, configs, **kwargs):
"""Build the pipelines to load data.
Args:
cycle_timestamp: String of timestamp, formatted as YYYYMMDDTHHMMSSZ.
configs: Dictionary of configurations.
kwargs: Extra configs.
Returns:
List of pipelines that will be run.
Raises: inventory_errors.LoadDataPipelineError.
"""
pipelines = []
# Commonly used clients for shared ratelimiter re-use.
crm_v1_api_client = crm.CloudResourceManagerClient()
dao = kwargs.get('dao')
gcs_api_client = gcs.StorageClient()
organization_dao_name = 'organization_dao'
project_dao_name = 'project_dao'
# The order here matters, e.g. groups_pipeline must come before
# group_members_pipeline.
pipelines = [
load_orgs_pipeline.LoadOrgsPipeline(cycle_timestamp, configs,
crm_v1_api_client,
kwargs.get(organization_dao_name)),
load_org_iam_policies_pipeline.LoadOrgIamPoliciesPipeline(
cycle_timestamp, configs, crm_v1_api_client,
kwargs.get(organization_dao_name)),
load_projects_pipeline.LoadProjectsPipeline(
cycle_timestamp, configs, crm_v1_api_client,
kwargs.get(project_dao_name)),
load_projects_iam_policies_pipeline.LoadProjectsIamPoliciesPipeline(
cycle_timestamp, configs, crm_v1_api_client,
kwargs.get(project_dao_name)),
load_projects_buckets_pipeline.LoadProjectsBucketsPipeline(
cycle_timestamp, configs, gcs_api_client,
kwargs.get(project_dao_name)),
load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline(
cycle_timestamp, configs, gcs_api_client,
kwargs.get('bucket_dao')),
load_projects_cloudsql_pipeline.LoadProjectsCloudsqlPipeline(
cycle_timestamp, configs, cloudsql.CloudsqlClient(),
kwargs.get('cloudsql_dao')),
load_forwarding_rules_pipeline.LoadForwardingRulesPipeline(
cycle_timestamp, configs, compute.ComputeClient(),
kwargs.get('fwd_rules_dao')),
load_folders_pipeline.LoadFoldersPipeline(
cycle_timestamp, configs,
crm.CloudResourceManagerClient(version='v2beta1'), dao),
load_bigquery_datasets_pipeline.LoadBigQueryDatasetsPipeline(
cycle_timestamp, configs, bq.BigQueryClient(), dao),
load_firewall_rules_pipeline.LoadFirewallRulesPipeline(
cycle_timestamp, configs, compute.ComputeClient(version='beta'),
kwargs.get(project_dao_name)),
]
if configs.get('inventory_groups'):
if util.can_inventory_groups(configs):
admin_api_client = ad.AdminDirectoryClient()
pipelines.extend([
load_groups_pipeline.LoadGroupsPipeline(
cycle_timestamp, configs, admin_api_client, dao),
load_group_members_pipeline.LoadGroupMembersPipeline(
cycle_timestamp, configs, admin_api_client, dao)
])
else:
raise inventory_errors.LoadDataPipelineError(
'Unable to inventory groups with specified arguments:\n%s',
configs)
return pipelines
def test_no_quota(self, mock_google_credential):
"""Verify no rate limiter is used if the configuration is missing."""
gce_api_client = compute.ComputeClient(global_configs={})
self.assertEqual(None, gce_api_client.repository._rate_limiter)
def setUpClass(cls, mock_google_credential):
"""Set up."""
fake_global_configs = {'max_compute_api_calls_per_second': 2000}
cls.gce_api_client = compute.ComputeClient(
global_configs=fake_global_configs)
cls.project_id = fake_compute.FAKE_PROJECT_ID
def setUp(self, mock_google_credential, mock_discovery):
"""Set up."""
fake_global_configs = {'max_compute_api_calls_per_second': 20}
self.client = compute.ComputeClient(global_configs=fake_global_configs)