def setup_new_project(config, starting_step, output_yaml_path):
"""Run the full process for initalizing a single new project.
Args:
config (ProjectConfig): The config of a single project to setup.
starting_step (int): The step number (indexed from 1) in _SETUP_STEPS to
begin from.
output_yaml_path (str): Path to output resulting root config in JSON.
Returns:
A boolean, true if the project was deployed successfully, false otherwise.
"""
steps = _SETUP_STEPS + config.extra_steps
total_steps = len(steps)
for step_num in range(starting_step, total_steps + 1):
logging.info('Step %s/%s', step_num, total_steps)
try:
steps[step_num - 1](config)
except subprocess.CalledProcessError as e:
logging.error('Setup failed on step %s: %s', step_num, e)
logging.error(
'To continue the script, sync the input file with the output file at '
'--output_yaml_path and re run the script with additional flags: '
'--resume_from_project=%s --resume_from_step=%s',
config.project['project_id'], step_num)
return False
utils.write_yaml_file(config.root, output_yaml_path)
logging.info('Setup completed successfully.')
return True
def _write_rules(deployment_config, directory):
"""Write a rules yaml file for each generator to the given directory."""
project_configs, global_config = get_all_project_configs(deployment_config)
for generator in SCANNER_RULE_GENERATORS:
config_file_name = generator.config_file_name()
logging.info('Generating rules for %s', config_file_name)
rules = generator.generate_rules(project_configs, global_config)
path = os.path.join(directory, config_file_name)
utils.write_yaml_file(rules, path)
def write_generated_fields(config):
"""Write gen fields to --generated_fields_path if set, else --project_yaml."""
if FLAGS.generated_fields_path:
utils.write_yaml_file(config.generated_fields,
FLAGS.generated_fields_path)
else:
config.root['generated_fields'] = config.generated_fields
field_generation.rewrite_generated_fields_back(FLAGS.project_yaml,
config.root)
def move_generated_fields_out_of_projects(input_yaml_path):
"""Move generated_fields out of projects."""
overall = utils.load_config(input_yaml_path)
if GENERATED_FIELDS_NAME in overall:
if is_old_generated_fields_format_exist(overall):
raise utils.InvalidConfigError(
'Generated fields conflict between new and old format.')
return False
convert_old_generated_fields_to_new(overall)
if GENERATED_FIELDS_NAME in overall:
if utils.wait_for_yes_no(
'Move generated_fields out of projects [y/N]?'):
utils.write_yaml_file(overall, input_yaml_path)
return True
return False
def setup_project(config, starting_step, output_yaml_path):
"""Run the full process for initalizing a single new project.
Note: for projects that have already been deployed, only the updatable steps
will be run.
Args:
config (ProjectConfig): The config of a single project to setup.
starting_step (int): The step number (indexed from 1) in _SETUP_STEPS to
begin from.
output_yaml_path (str): Path to output resulting root config in JSON.
Returns:
A boolean, true if the project was deployed successfully, false otherwise.
"""
steps = _SETUP_STEPS + config.extra_steps
deployed = is_deployed(config.project)
if deployed and not FLAGS.allow_updates:
logging.fatal('must pass --allow_updates to support updates.')
total_steps = len(steps)
for step_num in range(starting_step, total_steps + 1):
logging.info('Step %s/%s', step_num, total_steps)
step = steps[step_num - 1]
if deployed and not step.updatable:
logging.info('Step %s/%s is not updatable, skipping', step_num,
total_steps)
continue
try:
step.func(config)
except subprocess.CalledProcessError as e:
logging.error('Setup failed on step %s: %s', step_num, e)
logging.error(
'To continue the script, sync the input file with the output file at '
'--output_yaml_path and re run the script with additional flags: '
'--resume_from_project=%s --resume_from_step=%s',
config.project['project_id'], step_num)
return False
utils.write_yaml_file(config.root, output_yaml_path)
logging.info('Setup completed successfully.')
return True
def write_generated_fields(config):
"""Write gen fields to --generated_fields_path."""
utils.write_yaml_file(config.generated_fields, FLAGS.generated_fields_path)
def setup_project(config, output_yaml_path, output_cleanup_path):
"""Run the full process for initalizing a single new project.
Note: for projects that have already been deployed, only the updatable steps
will be run.
Args:
config (ProjectConfig): The config of a single project to setup.
output_yaml_path (str): Path to output resulting root config in JSON.
output_cleanup_path (str): Path to output cleanup shell script.
Returns:
A boolean, true if the project was deployed successfully, false otherwise.
"""
steps = _SETUP_STEPS + config.extra_steps
starting_step = config.project.get(_GENERATED_FIELDS_NAME, {}).get(
'failed_step', 1)
deployed = is_deployed(config.project)
total_steps = len(steps)
for step_num in range(starting_step, total_steps + 1):
step = steps[step_num - 1]
logging.info('%s: step %d/%d (%s)', config.project['project_id'], step_num,
total_steps, step.description)
if deployed and not step.updatable:
logging.info('Step %d is not updatable, skipping', step_num)
continue
try:
output = step.func(config)
if output and output.cleanup_commands:
with open(output_cleanup_path, 'a') as f:
for cmd in output.cleanup_commands:
f.write('# {}\n'.format(cmd))
except Exception as e: # pylint: disable=broad-except
logging.error('%s: setup failed on step %s: %s',
config.project['project_id'], step_num, e)
logging.error(
'Failure information has been written to --output_yaml_path. '
'Please ensure the config at --project_yaml is updated with any '
'changes from the config at --output_yaml_path and re-run the script'
'(Note: only applicable if --output_yaml_path != --project_yaml)')
# only record failed step if project was undeployed, an update can always
# start from the beginning
if not deployed:
generated_fields = config.project.get('generated_fields')
if not generated_fields:
generated_fields = {}
config.project['generated_fields'] = generated_fields
generated_fields['failed_step'] = step_num
utils.write_yaml_file(config.root, output_yaml_path)
return False
utils.write_yaml_file(config.root, output_yaml_path)
# if this deployment was resuming from a previous failure, remove the
# failed step as it is done
config.project.get(_GENERATED_FIELDS_NAME, {}).pop('failed_step', None)
utils.write_yaml_file(config.root, output_yaml_path)
logging.info('Setup completed successfully.')
return True
def main(argv):
del argv # Unused.
input_yaml_path = utils.normalize_path(FLAGS.project_yaml)
output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
output_rules_path = (utils.normalize_path(FLAGS.output_rules_path)
if FLAGS.output_rules_path else None)
# Output YAML will rearrange fields and remove comments, so do a basic check
# against accidental overwriting.
if input_yaml_path == output_yaml_path:
logging.error('output_yaml_path cannot overwrite project_yaml.')
return
# Read and parse the project configuration YAML file.
root_config = utils.resolve_env_vars(utils.read_yaml_file(input_yaml_path))
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
overall = root_config['overall']
audit_logs_project = root_config.get('audit_logs_project')
projects = []
# Always deploy the remote audit logs project first (if present).
if audit_logs_project:
projects.append(ProjectConfig(overall=overall,
project=audit_logs_project,
audit_logs_project=None))
forseti_config = root_config.get('forseti', {})
if forseti_config:
forseti_project_config = ProjectConfig(
overall=overall,
project=forseti_config.get('project'),
audit_logs_project=audit_logs_project)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
projects.append(ProjectConfig(overall=overall,
project=project_config,
audit_logs_project=audit_logs_project))
validate_project_configs(overall, projects)
# If resuming setup from a particular project, skip to that project.
if FLAGS.resume_from_project:
while (projects and
projects[0].project['project_id'] != FLAGS.resume_from_project):
skipped = projects.pop(0)
logging.info('Skipping project %s', skipped.project['project_id'])
if not projects:
logging.error('Project not found: %s', FLAGS.resume_from_project)
if projects:
starting_step = max(1, FLAGS.resume_from_step)
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_new_project(config, starting_step):
# Don't attempt to deploy additional projects if one project failed.
return
# Fill in unset generated fields for the project and save it.
add_generated_fields(config.project)
utils.write_yaml_file(root_config, output_yaml_path)
starting_step = 1
else:
logging.error('No projects to deploy.')
# TODO: allow for forseti installation to be retried.
if forseti_config:
forseti_project_id = forseti_config['project']['project_id']
forseti_service_account = forseti_config.get('generated_fields',
{}).get('service_account')
# If the forseti block does not have generated fields from a previous
# deployment, consider Forseti to be undeployed.
# TODO: add more checks of a previous deployment.
if 'generated_fields' not in forseti_config:
forseti.install(forseti_config)
forseti_service_account = forseti.get_server_service_account(
forseti_project_id)
forseti_config['generated_fields'] = {
'service_account': forseti_service_account,
'server_bucket': forseti.get_server_bucket(forseti_project_id),
}
utils.write_yaml_file(root_config, output_yaml_path)
for project in projects:
project_id = project.project['project_id']
forseti.grant_access(project_id,
forseti_service_account)
rule_generator.run(root_config, output_path=output_rules_path)
def rewrite_generated_fields_back(project_yaml, output_yaml_path, new_config):
"""Write config file to output_yaml_path with new generated_fields."""
cfg_content = update_generated_fields(project_yaml, new_config)
utils.write_yaml_file(cfg_content, output_yaml_path)
