def _output_results_to_db(self, resource_name, violations):
"""Output scanner results to DB.
Args:
resource_name (str): Resource name.
violations (list): A list of violations.
Returns:
list: Violations that encountered an error during insert.
"""
resource_name = 'violations'
(inserted_row_count, violation_errors) = (0, [])
try:
vdao = violation_dao.ViolationDao(self.global_configs)
(inserted_row_count, violation_errors) = vdao.insert_violations(
violations,
resource_name=resource_name,
snapshot_timestamp=self.snapshot_timestamp)
except db_errors.MySQLError as err:
LOGGER.error('Error importing violations to database: %s', err)
# TODO: figure out what to do with the errors. For now, just log it.
LOGGER.debug('Inserted %s rows with %s errors', inserted_row_count,
len(violation_errors))
return violation_errors
def __init__(self, resource, cycle_timestamp,
violations, notifier_config, pipeline_config):
"""Constructor for the base pipeline.
Args:
resource: violation resource name
cycle_timestamp: String of timestamp, formatted as YYYYMMDDTHHMMSSZ.
violations: Dictonary of violations
notifier_config: Dictionary of notifier configurations.
pipeline_config: Dictonary of pipeline confogurations.
Returns:
None
"""
self.cycle_timestamp = cycle_timestamp
self.resource = resource
self.notifier_config = notifier_config
self.pipeline_config = pipeline_config
# TODO: import api_client
# self.api_client = api_client
# Initializing DAOs
self.dao = dao.Dao()
self.project_dao = project_dao.ProjectDao()
self.violation_dao = violation_dao.ViolationDao()
# Get violations
self.violations = violations
def __init__(self, resource, cycle_timestamp, violations, global_configs,
notifier_config, pipeline_config):
"""Constructor for the base pipeline.
Args:
resource (str): Violation resource name.
cycle_timestamp (str): Snapshot timestamp,
formatted as YYYYMMDDTHHMMSSZ.
violations (dict): Violations.
global_configs (dict): Global configurations.
notifier_config (dict): Notifier configurations.
pipeline_config (dict): Pipeline configurations.
"""
self.cycle_timestamp = cycle_timestamp
self.resource = resource
self.global_configs = global_configs
self.notifier_config = notifier_config
self.pipeline_config = pipeline_config
# TODO: import api_client
# self.api_client = api_client
# Initializing DAOs
self.dao = dao.Dao(global_configs)
self.project_dao = project_dao.ProjectDao(global_configs)
self.violation_dao = violation_dao.ViolationDao(global_configs)
# Get violations
self.violations = violations
def main(_):
"""Main function.
Args:
_ (obj): Result of the last expression evaluated in the interpreter.
"""
if FLAGS.timestamp is not None:
timestamp = FLAGS.timestamp
else:
timestamp = _get_timestamp()
if FLAGS.config is None:
LOGGER.error('You must specify a notification pipeline')
exit()
notifier_configs = FLAGS.FlagValuesDict()
configs = file_loader.read_and_parse_file(FLAGS.config)
# get violations
v_dao = violation_dao.ViolationDao()
violations = {}
for resource in RESOURCE_MAP:
try:
violations[resource] = v_dao.get_all_violations(
timestamp, RESOURCE_MAP[resource])
except db_errors.MySQLError, e:
# even if an error is raised we still want to continue execution
# this is because if we don't have violations the Mysql table
# is not present and an error is thrown
LOGGER.error('get_all_violations error: %s', e.message)
def _output_results(all_violations, snapshot_timestamp, **kwargs):
"""Send the output results.
Args:
all_violations: The list of violations to report.
snapshot_timestamp: The snapshot timetamp associated with this scan.
**kwargs: The rest of the args.
"""
# Write violations to database.
flattening_scheme = kwargs.get('flattening_scheme')
resource_name = sm.RESOURCE_MAP[flattening_scheme]
(inserted_row_count, violation_errors) = (0, [])
try:
vdao = violation_dao.ViolationDao()
(inserted_row_count, violation_errors) = vdao.insert_violations(
all_violations,
resource_name=resource_name,
snapshot_timestamp=snapshot_timestamp)
except db_errors.MySQLError as err:
LOGGER.error('Error importing violations to database: %s', err)
# TODO: figure out what to do with the errors. For now, just log it.
LOGGER.debug('Inserted %s rows with %s errors', inserted_row_count,
len(violation_errors))
# Write the CSV for all the violations.
if FLAGS.output_path:
output_csv_name = None
with csv_writer.write_csv(resource_name=flattening_scheme,
data=_flatten_violations(
all_violations, flattening_scheme),
write_header=True) as csv_file:
output_csv_name = csv_file.name
LOGGER.info('CSV filename: %s', output_csv_name)
# Scanner timestamp for output file and email.
now_utc = datetime.utcnow()
output_path = FLAGS.output_path
if not output_path.startswith('gs://'):
if not os.path.exists(FLAGS.output_path):
os.makedirs(output_path)
output_path = os.path.abspath(output_path)
_upload_csv(output_path, now_utc, output_csv_name)
# Send summary email.
if FLAGS.email_recipient is not None:
email_pipeline = (
email_scanner_summary_pipeline.EmailScannerSummaryPipeline(
FLAGS.sendgrid_api_key))
email_pipeline.run(output_csv_name,
_get_output_filename(now_utc), now_utc,
all_violations,
kwargs.get('resource_counts',
{}), violation_errors,
FLAGS.email_sender, FLAGS.email_recipient)
def _get_violation_dao(self):
"""Init or get violation dao.
Returns:
violation_dao: ViolationDao instance
"""
if not self.violation_dao:
self.violation_dao = violation_dao.ViolationDao(
self.global_configs)
return self.violation_dao
def main(_):
"""main function"""
if FLAGS.timestamp is not None:
timestamp = FLAGS.timestamp
else:
timestamp = _get_timestamp()
if FLAGS.config is None:
LOGGER.error('You must specify a notification pipeline')
exit()
notifier_configs = FLAGS.FlagValuesDict()
configs = file_loader.read_and_parse_file(FLAGS.config)
# get violations
v_dao = violation_dao.ViolationDao()
violations = {
'violations':
v_dao.get_all_violations(timestamp, 'violations'),
'bucket_acl_violations':
v_dao.get_all_violations(timestamp, 'buckets_acl_violations')
}
for retrieved_v in violations:
LOGGER.info('retrieved %d violations for resource \'%s\'',
len(violations[retrieved_v]), retrieved_v)
# build notification pipelines
pipelines = []
for resource in configs['resources']:
if violations.get(resource['resource']) is None:
LOGGER.error('The resource name \'%s\' is invalid, skipping',
resource['resource'])
continue
if resource['should_notify'] is False:
continue
for pipeline in resource['pipelines']:
LOGGER.info('Running \'%s\' pipeline for resource \'%s\'',
pipeline['name'], resource['resource'])
chosen_pipeline = find_pipelines(pipeline['name'])
pipelines.append(
chosen_pipeline(resource['resource'], timestamp,
violations[resource['resource']],
notifier_configs, pipeline['configuration']))
# run the pipelines
for pipeline in pipelines:
pipeline.run()
def main(_):
"""Main function.
Args:
_ (obj): Result of the last expression evaluated in the interpreter.
"""
notifier_flags = FLAGS.FlagValuesDict()
forseti_config = notifier_flags.get('forseti_config')
if forseti_config is None:
LOGGER.error('Path to Forseti Security config needs to be specified.')
sys.exit()
try:
configs = file_loader.read_and_parse_file(forseti_config)
except IOError:
LOGGER.error('Unable to open Forseti Security config file. '
'Please check your path and filename and try again.')
sys.exit()
global_configs = configs.get('global')
notifier_configs = configs.get('notifier')
timestamp = notifier_configs.get('timestamp')
if timestamp is None:
timestamp = _get_timestamp(global_configs)
# get violations
v_dao = violation_dao.ViolationDao(global_configs)
violations_as_dict = v_dao.get_all_violations(timestamp)
for i in violations_as_dict:
i['created_at_datetime'] = (
i.get('created_at_datetime').strftime('%Y-%m-%dT%H:%M:%SZ'))
violations = {}
try:
violations = violation_dao.map_by_resource(violations_as_dict)
except db_errors.MySQLError, e:
# even if an error is raised we still want to continue execution
# this is because if we don't have violations the Mysql table
# is not present and an error is thrown
LOGGER.error('get_all_violations error: %s', e.message)
def setUp(self, mock_db_connector):
mock_db_connector.return_value = None
self.resource_name = 'violations'
self.dao = violation_dao.ViolationDao()
self.fake_snapshot_timestamp = '12345'
self.fake_table_name = (
'%s_%s' % (self.resource_name, self.fake_snapshot_timestamp))
self.fake_violations = [
rules.RuleViolation(
resource_type='x',
resource_id='1',
rule_name='rule name',
rule_index=0,
violation_type='ADDED',
role='roles/editor',
members=[
iam.IamPolicyMember.create_from(m)
for m in ['user:[email protected]', 'user:[email protected]']
],
),
rules.RuleViolation(
resource_type='%sb' % ('a' * 300),
resource_id='1',
rule_name='%sd' % ('c' * 300),
rule_index=1,
violation_type='REMOVED',
role='%s' % ('e' * 300),
members=[
iam.IamPolicyMember.create_from('user:%sh' % ('g' * 300))
],
),
]
self.expected_fake_violations = [
('x', '1', 'rule name', 0, 'ADDED', 'roles/editor',
'user:[email protected]'),
('x', '1', 'rule name', 0, 'ADDED', 'roles/editor',
'user:[email protected]'),
('a' * 255, '1', 'c' * 255, 1, 'REMOVED', 'e' * 255,
('user:%s' % ('g' * 300))[:255]),
]
def setUp(self, mock_db_connector):
mock_db_connector.return_value = None
self.resource_name = 'violations'
self.dao = violation_dao.ViolationDao()
self.fake_snapshot_timestamp = '12345'
self.fake_table_name = (
'%s_%s' % (self.resource_name, self.fake_snapshot_timestamp))
self.fake_violations = [
rules.RuleViolation(
resource_type='x',
resource_id='1',
rule_name='rule name',
rule_index=0,
violation_type='ADDED',
role='roles/editor',
members=[
iam.IamPolicyMember.create_from(m)
for m in ['user:[email protected]', 'user:[email protected]']
],
),
rules.RuleViolation(
resource_type='%se' % ('a' * 300),
resource_id='1',
rule_name='%sh' % ('b' * 300),
rule_index=1,
violation_type='REMOVED',
role='%s' % ('c' * 300),
members=[
iam.IamPolicyMember.create_from('user:%s' % ('d' * 300))
],
),
]
long_string = '{"member": "user:%s", "role": "%s"}' % (('d' * 300),
('c' * 300))
self.fake_flattened_violations = [
{
'resource_id': '1',
'resource_type': self.fake_violations[0].resource_type,
'rule_index': 0,
'rule_name': self.fake_violations[0].rule_name,
'violation_type': self.fake_violations[0].violation_type,
'violation_data': {
'role': self.fake_violations[0].role,
'member': 'user:[email protected]'
}
},
{
'resource_id': '1',
'resource_type': self.fake_violations[0].resource_type,
'rule_index': 0,
'rule_name': self.fake_violations[0].rule_name,
'violation_type': self.fake_violations[0].violation_type,
'violation_data': {
'role': self.fake_violations[0].role,
'member': 'user:[email protected]'
}
},
{
'resource_id': '1',
'resource_type': self.fake_violations[1].resource_type,
'rule_index': 1,
'rule_name': self.fake_violations[1].rule_name,
'violation_type': self.fake_violations[1].violation_type,
'violation_data': {
'role': self.fake_violations[1].role,
'member': 'user:%s' % ('d' * 300)
}
},
]
self.expected_fake_violations = [
('x', '1', 'rule name', 0, 'ADDED',
'{"member": "user:[email protected]", "role": "roles/editor"}'),
('x', '1', 'rule name', 0, 'ADDED',
'{"member": "user:[email protected]", "role": "roles/editor"}'),
('a' * 255, '1', 'b' * 255, 1, 'REMOVED', long_string),
]
def _output_results(all_violations, snapshot_timestamp, **kwargs):
"""Send the output results.
Args:
all_violations: The list of violations to report.
snapshot_timestamp: The snapshot timetamp associated with this scan.
**kwargs: The rest of the args.
"""
# Write violations to database.
flattening_scheme = kwargs.get('flattening_scheme')
resource_name = sm.RESOURCE_MAP[flattening_scheme]
(inserted_row_count, violation_errors) = (0, [])
all_violations = _flatten_violations(all_violations, flattening_scheme)
try:
vdao = violation_dao.ViolationDao()
(inserted_row_count, violation_errors) = vdao.insert_violations(
all_violations,
resource_name=resource_name,
snapshot_timestamp=snapshot_timestamp)
except db_errors.MySQLError as err:
LOGGER.error('Error importing violations to database: %s', err)
# TODO: figure out what to do with the errors. For now, just log it.
LOGGER.debug('Inserted %s rows with %s errors', inserted_row_count,
len(violation_errors))
# Write the CSV for all the violations.
if FLAGS.output_path:
LOGGER.info('Writing violations to csv...')
output_csv_name = None
with csv_writer.write_csv(resource_name=resource_name,
data=all_violations,
write_header=True) as csv_file:
output_csv_name = csv_file.name
LOGGER.info('CSV filename: %s', output_csv_name)
# Scanner timestamp for output file and email.
now_utc = datetime.utcnow()
output_path = FLAGS.output_path
if not output_path.startswith('gs://'):
if not os.path.exists(FLAGS.output_path):
os.makedirs(output_path)
output_path = os.path.abspath(output_path)
_upload_csv(output_path, now_utc, output_csv_name)
# Send summary email.
if FLAGS.email_recipient is not None:
payload = {
'email_sender': FLAGS.email_sender,
'email_recipient': FLAGS.email_recipient,
'sendgrid_api_key': FLAGS.sendgrid_api_key,
'output_csv_name': output_csv_name,
'output_filename': _get_output_filename(now_utc),
'now_utc': now_utc,
'all_violations': all_violations,
'resource_counts': kwargs.get('resource_counts', {}),
'violation_errors': violation_errors
}
message = {'status': 'scanner_done', 'payload': payload}
notifier.process(message)
def setUp(self, mock_db_connector):
mock_db_connector.return_value = None
self.resource_name = 'violations'
self.dao = violation_dao.ViolationDao()
self.fake_snapshot_timestamp = '12345'
self.fake_table_name = ('%s_%s' %
(self.resource_name, self.fake_snapshot_timestamp))
self.fake_violations = [
rules.RuleViolation(
resource_type='x',
resource_id='1',
rule_name='rule name',
rule_index=0,
violation_type='ADDED',
role='roles/editor',
members=[iam.IamPolicyMember.create_from(m)
for m in ['user:[email protected]', 'user:[email protected]']],
),
rules.RuleViolation(
resource_type='%se' % ('a'*300),
resource_id='1',
rule_name='%sh' % ('b'*300),
rule_index=1,
violation_type='REMOVED',
role='%s' % ('c'*300),
members=[iam.IamPolicyMember.create_from(
'user:%s' % ('d'*300))],
),
]
long_string = '{"member": "user:%s", "role": "%s"}' % (('d'*300),('c'*300))
self.fake_flattened_violations = [
{
'resource_id': '1',
'resource_type':
self.fake_violations[0].resource_type,
'rule_index': 0,
'rule_name': self.fake_violations[0].rule_name,
'violation_type': self.fake_violations[0].violation_type,
'violation_data': {
'role': self.fake_violations[0].role,
'member': 'user:[email protected]'
}
},
{
'resource_id': '1',
'resource_type':
self.fake_violations[0].resource_type,
'rule_index': 0,
'rule_name': self.fake_violations[0].rule_name,
'violation_type': self.fake_violations[0].violation_type,
'violation_data': {
'role': self.fake_violations[0].role,
'member': 'user:[email protected]'
}
},
{
'resource_id': '1',
'resource_type':
self.fake_violations[1].resource_type,
'rule_index': 1,
'rule_name': self.fake_violations[1].rule_name,
'violation_type': self.fake_violations[1].violation_type,
'violation_data': {
'role': self.fake_violations[1].role,
'member': 'user:%s' % ('d'*300)
}
},
]
self.expected_fake_violations = [
('2e598a34bf36b5ad577f6e8eec47acf0ffe71143e82bc78c9cfaac8f553a4fa4f09078ebe0769639d9dc9fffc2dffb94345ae6696fbe4e646e8b9bb723fd3ab0',
'x', '1', 'rule name', 0, 'ADDED',
'{"member": "user:[email protected]", "role": "roles/editor"}',
'2020-08-28 10:20:30'),
('5420235c547006300e7842c45c9b0419bc1c5590fd4200607e9925010123f5905f374c2b182420f6ecb161146c8d18c6683045026c810009154b03a56f5a94f5',
'x', '1', 'rule name', 0, 'ADDED',
'{"member": "user:[email protected]", "role": "roles/editor"}',
'2010-08-28 10:20:30'),
('i'*255,
'a'*255, '1', 'b'*255, 1, 'REMOVED', long_string,
'2030-08-28 10:20:30'),
]
