def main(argv):
del argv # Unused.
deployment_config = utils.read_yaml_file(
utils.normalize_path(FLAGS.deployment_config_path))
output_path = (utils.normalize_path(FLAGS.output_path)
if FLAGS.output_path else None)
rule_generator.run(deployment_config, output_path=output_path)
def test_load_config_spanned_configs(self):
project_yaml = ('deploy/samples/'
'project_with_remote_audit_logs.yaml')
input_yaml_path = utils.normalize_path(project_yaml)
dict1 = utils.load_config(input_yaml_path)
project_yaml = ('deploy/samples/spanned_configs/root.yaml')
input_yaml_path = utils.normalize_path(project_yaml)
dict2 = utils.load_config(input_yaml_path)
self.assertTrue(is_expand_config_equal(dict1, dict2))
def main(argv):
del argv # Unused.
input_yaml_path = utils.normalize_path(FLAGS.project_yaml)
output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
output_rules_path = None
if FLAGS.output_rules_path:
output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
# Output YAML will rearrange fields and remove comments, so do a basic check
# against accidental overwriting.
if input_yaml_path == output_yaml_path:
logging.error('output_yaml_path cannot overwrite project_yaml.')
return
# Read and parse the project configuration YAML file.
root_config = utils.load_config(input_yaml_path)
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
audit_logs_project = root_config.get('audit_logs_project')
projects = []
# Always deploy the remote audit logs project first (if present).
if not is_deployed(audit_logs_project):
projects.append(
ProjectConfig(root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[]))
forseti_config = root_config.get('forseti', {})
if not is_deployed(forseti_config.get('project')):
extra_steps = [
install_forseti,
get_forseti_access_granter(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter(audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if is_deployed(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter(project_config['project_id']))
projects.append(
ProjectConfig(root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
starting_step = 1
if config.project['project_id'] == FLAGS.resume_from_project:
starting_step = max(1, FLAGS.resume_from_step)
if not setup_new_project(config, starting_step, output_yaml_path):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
rule_generator.run(root_config, output_path=output_rules_path)
def main(argv):
del argv # Unused.
if FLAGS.enable_new_style_resources:
logging.info('--enable_new_style_resources is true.')
FLAGS.output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
FLAGS.output_cleanup_path = utils.normalize_path(FLAGS.output_cleanup_path)
if FLAGS.output_rules_path:
FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml)
if field_generation.move_generated_fields_out_of_projects(FLAGS.project_yaml):
if FLAGS.dry_run:
logging.error(
'Must convert generated fields in nodry_run before running!')
return
elif not utils.wait_for_yes_no(
'Use converted generated fields to continue? [y/N]?'):
return
# Read and parse the project configuration YAML file.
root_config = utils.load_config(FLAGS.project_yaml)
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
with open(FLAGS.output_cleanup_path, 'w') as f:
f.write(_CLEANUP_HEADER)
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(
root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[]))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(
root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config, FLAGS.project_yaml, FLAGS.output_yaml_path,
FLAGS.output_cleanup_path):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
if FLAGS.enable_new_style_resources:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
subprocess.check_call(call)
else:
rule_generator.run(root_config, output_path=FLAGS.output_rules_path)
logging.info(
'All projects successfully deployed. Please remember to sync '
'any changes written to the config at --output_yaml_path with '
'--project_yaml before running the script again (Note: only applicable '
'if --output_yaml_path != --project_yaml)')
def main(argv):
del argv # Unused.
if FLAGS.generated_fields_path == FLAGS.project_yaml:
raise Exception(
'--generated_fields_path must not be set to the same as --project_yaml.'
)
if FLAGS.output_rules_path:
FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml)
FLAGS.generated_fields_path = utils.normalize_path(
FLAGS.generated_fields_path)
# touch file if it has not been created yet
open(FLAGS.generated_fields_path, 'a').close()
config_string = runner.run_command([
FLAGS.load_config_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--generated_fields_path',
FLAGS.generated_fields_path,
],
get_output=True)
root_config = yaml.load(config_string)
if not root_config:
logging.error('Error loading project YAML.')
return
generated_fields = utils.read_yaml_file(FLAGS.generated_fields_path)
if not generated_fields:
generated_fields = {'projects': {}}
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[],
generated_fields=generated_fields))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(
audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps,
generated_fields=generated_fields)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps,
generated_fields=generated_fields))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--generated_fields_path',
FLAGS.generated_fields_path,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
utils.call_go_binary(call)
logging.info('All projects successfully deployed.')
def main(argv):
del argv # Unused.
input_yaml_path = utils.normalize_path(FLAGS.project_yaml)
output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
output_cleanup_path = utils.normalize_path(FLAGS.output_cleanup_path)
output_rules_path = None
if FLAGS.output_rules_path:
output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
# Read and parse the project configuration YAML file.
root_config = utils.load_config(input_yaml_path)
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
with open(output_cleanup_path, 'w') as f:
f.write(_CLEANUP_HEADER)
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(
root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[]))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(
root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config, output_yaml_path, output_cleanup_path):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
rule_generator.run(root_config, output_path=output_rules_path)
logging.info(
'All projects successfully deployed. Please remember to sync '
'any changes written to the config at --output_yaml_path with '
'--project_yaml before running the script again (Note: only applicable '
'if --output_yaml_path != --project_yaml)')
def main(argv):
del argv # Unused.
input_yaml_path = utils.normalize_path(FLAGS.project_yaml)
output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
output_rules_path = (utils.normalize_path(FLAGS.output_rules_path)
if FLAGS.output_rules_path else None)
# Output YAML will rearrange fields and remove comments, so do a basic check
# against accidental overwriting.
if input_yaml_path == output_yaml_path:
logging.error('output_yaml_path cannot overwrite project_yaml.')
return
# Read and parse the project configuration YAML file.
root_config = utils.resolve_env_vars(utils.read_yaml_file(input_yaml_path))
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
overall = root_config['overall']
audit_logs_project = root_config.get('audit_logs_project')
projects = []
# Always deploy the remote audit logs project first (if present).
if audit_logs_project:
projects.append(ProjectConfig(overall=overall,
project=audit_logs_project,
audit_logs_project=None))
forseti_config = root_config.get('forseti', {})
if forseti_config:
forseti_project_config = ProjectConfig(
overall=overall,
project=forseti_config.get('project'),
audit_logs_project=audit_logs_project)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
projects.append(ProjectConfig(overall=overall,
project=project_config,
audit_logs_project=audit_logs_project))
validate_project_configs(overall, projects)
# If resuming setup from a particular project, skip to that project.
if FLAGS.resume_from_project:
while (projects and
projects[0].project['project_id'] != FLAGS.resume_from_project):
skipped = projects.pop(0)
logging.info('Skipping project %s', skipped.project['project_id'])
if not projects:
logging.error('Project not found: %s', FLAGS.resume_from_project)
if projects:
starting_step = max(1, FLAGS.resume_from_step)
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_new_project(config, starting_step):
# Don't attempt to deploy additional projects if one project failed.
return
# Fill in unset generated fields for the project and save it.
add_generated_fields(config.project)
utils.write_yaml_file(root_config, output_yaml_path)
starting_step = 1
else:
logging.error('No projects to deploy.')
# TODO: allow for forseti installation to be retried.
if forseti_config:
forseti_project_id = forseti_config['project']['project_id']
forseti_service_account = forseti_config.get('generated_fields',
{}).get('service_account')
# If the forseti block does not have generated fields from a previous
# deployment, consider Forseti to be undeployed.
# TODO: add more checks of a previous deployment.
if 'generated_fields' not in forseti_config:
forseti.install(forseti_config)
forseti_service_account = forseti.get_server_service_account(
forseti_project_id)
forseti_config['generated_fields'] = {
'service_account': forseti_service_account,
'server_bucket': forseti.get_server_bucket(forseti_project_id),
}
utils.write_yaml_file(root_config, output_yaml_path)
for project in projects:
project_id = project.project['project_id']
forseti.grant_access(project_id,
forseti_service_account)
rule_generator.run(root_config, output_path=output_rules_path)
def main(argv):
del argv # Unused.
if FLAGS.enable_new_style_resources:
logging.info('--enable_new_style_resources is true.')
FLAGS.output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
if FLAGS.output_rules_path:
FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml)
if FLAGS.enable_new_style_resources:
config_string = runner.run_command([
FLAGS.load_config_binary,
'--config_path',
FLAGS.project_yaml,
],
get_output=True)
yaml = ruamel.yaml.YAML()
root_config = yaml.load(config_string)
else:
root_config = utils.load_config(FLAGS.project_yaml)
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(
root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[]))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(
root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config, FLAGS.project_yaml, FLAGS.output_yaml_path):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
if FLAGS.enable_new_style_resources:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
utils.call_go_binary(call)
else:
rule_generator.run(root_config, output_path=FLAGS.output_rules_path)
logging.info(
'All projects successfully deployed. Please remember to sync '
'any changes written to the config at --output_yaml_path with '
'--project_yaml before running the script again (Note: only applicable '
'if --output_yaml_path != --project_yaml)')
