def insert_violations(self,
violations,
resource_name,
snapshot_timestamp=None):
"""Import violations into database.
Args:
violations: An iterator of RuleViolations.
resource_name: String that defines a resource
snapshot_timestamp: The snapshot timestamp to associate these
violations with.
Return:
A tuple of (int, list) containing the count of inserted rows and
a list of violations that encountered an error during insert.
Raise:
MySQLError if snapshot table could not be created.
"""
try:
# Make sure to have a reasonable timestamp to use.
if not snapshot_timestamp:
snapshot_timestamp = self.get_latest_snapshot_timestamp(
('PARTIAL_SUCCESS', 'SUCCESS'))
# Create the violations snapshot table.
snapshot_table = self._create_snapshot_table(
resource_name, snapshot_timestamp)
except MySQLdb.Error, e:
raise db_errors.MySQLError(resource_name, e)
def test_retrieve_error_raised_when_db_error(self):
"""Test that LoadDataPipelineError is raised when database error."""
self.mock_dao.get_folders.side_effect = (data_access_errors.MySQLError(
'folders', mock.MagicMock()))
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._retrieve()
def get_buckets_acls(self, resource_name, timestamp):
"""Select the bucket acls from a bucket acls snapshot table.
Args:
resource_name: String of the resource name.
timestamp: String of timestamp, formatted as YYYYMMDDTHHMMSSZ.
Returns:
List of bucket acls.
Raises:
MySQLError: An error with MySQL has occurred.
"""
bucket_acls = {}
cnt = 0
try:
bucket_acls_sql = select_data.BUCKET_ACLS.format(timestamp)
rows = self.execute_sql_with_fetch(resource_name,
bucket_acls_sql,
None)
for row in rows:
bucket_acl = bkt_acls.\
BucketAccessControls(bucket=row['bucket'],
entity=row['entity'],
email=row['email'],
domain=row['domain'],
role=row['role'],
project_number=row['project_number'])
bucket_acls[cnt] = bucket_acl
cnt += 1
except (DataError, IntegrityError, InternalError, NotSupportedError,
OperationalError, ProgrammingError) as e:
LOGGER.error(errors.MySQLError(resource_name, e))
return bucket_acls
def _get_cloudsql_instance_acl_map(self, resource_name, timestamp):
"""Create CloudSQL instance acl map.
Args:
resource_name: String of the resource name.
timestamp: String of timestamp, formatted as YYYYMMDDTHHMMSSZ.
Returns:
Map of instance acls.
Raises:
MySQLError: An error with MySQL has occurred.
"""
cloudsql_acls_map = {}
try:
cloudsql_acls_sql = select_data.CLOUDSQL_ACLS.format(timestamp)
rows = self.execute_sql_with_fetch(resource_name,
cloudsql_acls_sql, None)
for row in rows:
acl_list = []
project_number = row['project_number']
instance_name = row['instance_name']
network = row['value']
hash_key = hash(str(project_number) + ',' + instance_name)
if hash_key in cloudsql_acls_map:
if network not in cloudsql_acls_map[hash_key]:
cloudsql_acls_map[hash_key].append(network)
else:
acl_list.append(network)
cloudsql_acls_map[hash_key] = acl_list
except (DataError, IntegrityError, InternalError, NotSupportedError,
OperationalError, ProgrammingError) as e:
LOGGER.error(errors.MySQLError(resource_name, e))
return cloudsql_acls_map
def test_get_group_members_raises_error(self, mock_fetch):
"""Test get_group_members() raises error."""
mock_fetch.side_effect = errors.MySQLError(self.resource_name,
mock.MagicMock())
with self.assertRaises(errors.MySQLError):
self.dao.get_group_members(self.resource_name, self.fake_group_id,
self.fake_timestamp)
def test_dao_error_is_handled_when_retrieving(self):
"""Test that exceptions are handled when retrieving."""
self.pipeline.dao.get_project_numbers.side_effect = (
data_access_errors.MySQLError('error error', mock.MagicMock()))
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._retrieve()
def test_get_timestamp_db_errors(self, mock_get_ss_timestamp, mock_conn):
"""Test that get_timestamp() works."""
mock_get_ss_timestamp.side_effect = errors.MySQLError(
'snapshot_cycles', mock.MagicMock())
scanner.LOGGER = mock.MagicMock()
actual = scanner._get_timestamp()
self.assertEqual(1, scanner.LOGGER.error.call_count)
self.assertIsNone(actual)
def test_load_errors_are_handled(self):
"""Test that errors are handled when loading."""
self.pipeline.dao.load_data.side_effect = (
data_access_errors.MySQLError('error error', mock.MagicMock()))
with self.assertRaises(inventory_errors.LoadDataPipelineError):
self.pipeline._load(self.pipeline.RESOURCE_NAME,
fake_projects.EXPECTED_LOADABLE_PROJECTS)
def test_get_timestamp_handles_error(self, mock_get_ss_timestamp,
mock_conn):
"""Test that get_timestamp() works."""
mock_get_ss_timestamp.side_effect = errors.MySQLError(
'snapshot_cycles', mock.MagicMock())
actual = self.scanner._get_timestamp()
expected = None
self.assertEquals(expected, actual)
self.assertEquals(1, self.scanner.LOGGER.error.call_count)
def test_get_project_numbers_raises_error(self):
"""Test get_project_numbers() raises a MySQLError."""
self.fetch_mock.side_effect = (errors.MySQLError(
self.resource_name, mock.MagicMock()))
with self.assertRaises(errors.MySQLError):
self.bucket_dao.get_buckets_by_project_number(
self.resource_name, self.fake_timestamp,
self.FAKE_PROJECT_NUMBERS[0])
def test_get_cloudsql_instance_acl_map_raises_error(self):
"""Test _get_cloudsql_instance_acl_map() raises a MySQLError."""
fetch_mock = mock.MagicMock()
self.cloudsql_dao.execute_sql_with_fetch = fetch_mock
fetch_mock.side_effect = (errors.MySQLError(self.resource_name,
mock.MagicMock()))
with self.assertRaises(errors.MySQLError):
self.cloudsql_dao._get_cloudsql_instance_acl_map(
self.resource_name, self.fake_timestamp)
def test_error_is_handled_in_get_loaded_count(self):
"""Test error from get_loaded_count is handled."""
base_pipeline.LOGGER = mock.create_autospec(log_util).get_logger('foo')
self.pipeline.dao.select_record_count.side_effect = (
data_access_errors.MySQLError('11111', '22222'))
self.pipeline._get_loaded_count()
self.assertEquals(1, base_pipeline.LOGGER.error.call_count)
self.assertIsNone(self.pipeline.count)
class ViolationDao(dao.Dao):
"""Data access object (DAO) for rule violations."""
violation_attribute_list = [
'resource_type', 'resource_id', 'rule_name', 'rule_index',
'violation_type', 'violation_data'
]
frozen_violation_attribute_list = frozenset(violation_attribute_list)
Violation = namedtuple('Violation', frozen_violation_attribute_list)
def insert_violations(self,
violations,
resource_name,
snapshot_timestamp=None):
"""Import violations into database.
Args:
violations (iterator): An iterator of RuleViolations.
resource_name (str): String that defines a resource.
snapshot_timestamp (str): The snapshot timestamp to associate
these violations with.
Returns:
tuple: A tuple of (int, list) containing the count of inserted
rows and a list of violations that encountered an error during
insert.
Raise:
MySQLError: is raised when the snapshot table can not be created.
"""
try:
# Make sure to have a reasonable timestamp to use.
if not snapshot_timestamp:
snapshot_timestamp = self.get_latest_snapshot_timestamp(
('PARTIAL_SUCCESS', 'SUCCESS'))
# Create the violations snapshot table.
snapshot_table = self.create_snapshot_table(
resource_name, snapshot_timestamp)
# TODO: Remove this exception handling by moving the check for
# violations table outside of the scanners.
except MySQLdb.OperationalError, e:
if e[0] == 1050:
LOGGER.debug('Violations table already exists: %s', e)
snapshot_table = self._create_snapshot_table_name(
resource_name, snapshot_timestamp)
else:
raise db_errors.MySQLError(resource_name, e)
except MySQLdb.Error, e:
raise db_errors.MySQLError(resource_name, e)
def test_pipeline_no_rules_loads_nothing(
self, mock_get_projects, mock_conn):
"""Test the pipeline with no instance_groups."""
mock_get_projects.return_value = self.projects
base_pipeline.LOGGER = mock.MagicMock()
self.pipeline.api_client.get_instance_groups = mock.MagicMock(
side_effect=[[], []])
self.pipeline.dao.select_record_count = mock.MagicMock(
side_effect=data_access_errors.MySQLError(
'instance_groups', mock.MagicMock()))
self.pipeline.run()
self.assertEquals(None, self.pipeline.count)
def test_get_projects_raises_error_on_fetch_error(self):
"""Test get_projects() raises MySQLError on fetch error.
Setup:
Set execute_sql_with_fetch() side effect to MySQLError.
Expected:
get_projects() raises MySQLError.
"""
self.fetch_mock.side_effect = errors.MySQLError(
self.resource_name, mock.MagicMock())
with self.assertRaises(errors.MySQLError):
self.project_dao.get_projects(self.fake_timestamp)
def test_get_project_policies_query_failed_handles_error(self):
"""Test that a failed get_project_policies() handles the error.
Setup:
Set execute_sql_with_fetch() side effect to MySQLError.
Expect:
get_project_policies() raises a MySQLError.
"""
self.fetch_mock.side_effect = errors.MySQLError(
self.resource_name, mock.MagicMock())
with self.assertRaises(errors.MySQLError):
self.project_dao.get_project_policies(self.resource_name,
self.fake_timestamp)
def test_get_org_query_failed_raises_error(self):
"""Test that a failed get_organization() raises a MySQLError.
Setup:
Create magic mock for execute_sql_with_fetch().
Expect:
Raises a MySQLError.
"""
self.fetch_mock.side_effect = errors.MySQLError(
self.resource_name, mock.MagicMock())
org_id = self.fake_orgs_db_rows[0]['org_id']
with self.assertRaises(errors.MySQLError):
org = self.org_dao.get_organization(org_id, self.fake_timestamp)
def test_get_org_iam_policies_query_failed_handles_error(self):
"""Test that a failed get_org_iam_policies() handles the error.
Setup:
Create magic mock for execute_sql_with_fetch().
Expect:
Raises a MySQLError.
"""
self.fetch_mock.side_effect = errors.MySQLError(
self.resource_name, mock.MagicMock())
organization_dao.LOGGER = mock.MagicMock()
with self.assertRaises(errors.MySQLError):
self.org_dao.get_org_iam_policies(
self.resource_name, self.fake_timestamp)
def get_cloudsql_acls(self, resource_name, timestamp):
"""Select the cloudsql acls for project from a snapshot table.
Args:
resource_name (str): String of the resource name.
timestamp (str): String of timestamp, formatted as
YYYYMMDDTHHMMSSZ.
Returns:
list: List of cloudsql acls.
Raises:
MySQLError: An error with MySQL has occurred.
"""
cloudsql_acls = {}
cnt = 0
try:
cloudsql_instances_sql = (
select_data.CLOUDSQL_INSTANCES.format(timestamp))
rows = self.execute_sql_with_fetch(resource_name,
cloudsql_instances_sql, None)
acl_map = self._get_cloudsql_instance_acl_map(
resource_name, timestamp)
for row in rows:
project_number = row['project_number']
instance_name = row['name']
ssl_enabled = row['settings_ip_configuration_require_ssl']
authorized_networks = self.\
_get_networks_for_instance(acl_map,
project_number,
instance_name)
cloudsql_acl = csql_acls.\
CloudSqlAccessControl(instance_name=instance_name,
authorized_networks=authorized_networks,
ssl_enabled=ssl_enabled,
project_number=project_number)
cloudsql_acls[cnt] = cloudsql_acl
cnt += 1
except (DataError, IntegrityError, InternalError, NotSupportedError,
OperationalError, ProgrammingError) as e:
LOGGER.error(errors.MySQLError(resource_name, e))
return cloudsql_acls
def test_get_orgs_query_failed_returns_emptylist(self):
"""Test that a failed get_organizations() returns an empty list.
Setup:
Create magic mock for execute_sql_with_fetch().
Expect:
* get_organizations() raises MySQLError.
"""
self.fetch_mock.side_effect = errors.MySQLError(
self.resource_name, mock.MagicMock())
organization_dao.LOGGER = mock.MagicMock()
fake_query = select_data.ORGANIZATIONS.format(self.fake_timestamp)
with self.assertRaises(errors.MySQLError):
orgs = self.org_dao.get_organizations(
self.resource_name, self.fake_timestamp)
self.fetch_mock.assert_called_once_with(
self.resource_name, fake_query)
def insert_violations(self, violations,
snapshot_timestamp=None):
"""Import violations into database.
Args:
violations (iterator): An iterator of RuleViolations.
snapshot_timestamp (str): The snapshot timestamp to associate
these violations with.
Returns:
tuple: A tuple of (int, list) containing the count of inserted
rows and a list of violations that encountered an error during
insert.
Raise:
MySQLError: is raised when the snapshot table can not be created.
"""
resource_name = 'violations'
try:
# Make sure to have a reasonable timestamp to use.
if not snapshot_timestamp:
snapshot_timestamp = self.get_latest_snapshot_timestamp(
('PARTIAL_SUCCESS', 'SUCCESS'))
# Create the violations snapshot table.
snapshot_table = self.create_snapshot_table(
resource_name, snapshot_timestamp)
# TODO: Remove this exception handling by moving the check for
# violations table outside of the scanners.
except MySQLdb.OperationalError, e:
if e[0] == 1050:
LOGGER.debug('Violations table already exists: %s', e)
snapshot_table = self._create_snapshot_table_name(
resource_name, snapshot_timestamp)
else:
raise db_errors.MySQLError(resource_name, e)
def get_bigquery_acls(self, resource_name, timestamp):
"""Select the Big Query acls from a Big Query acls snapshot table.
Args:
resource_name (str): String of the resource name.
timestamp (str): String of timestamp, formatted as
YYYYMMDDTHHMMSSZ.
Returns:
dict: Dictionary keyed by the count of ACLs and then the ACLs.
Raises:
MySQLError: An error with MySQL has occurred.
"""
bigquery_acls = {}
cnt = 0
try:
bigquery_acls_sql = select_data.BIGQUERY_ACLS.format(timestamp)
rows = self.execute_sql_with_fetch(resource_name,
bigquery_acls_sql, None)
except (DataError, IntegrityError, InternalError, NotSupportedError,
OperationalError, ProgrammingError) as e:
LOGGER.error(errors.MySQLError(resource_name, e))
for row in rows:
bigquery_acl = bq_acls.BigqueryAccessControls(
dataset_id=row['dataset_id'],
special_group=row['access_special_group'],
user_email=row['access_user_by_email'],
domain=row['access_domain'],
role=row['role'],
group_email=row['access_group_by_email'],
project_id=row['project_id'])
bigquery_acls[cnt] = bigquery_acl
cnt += 1
return bigquery_acls
