def main(argv): del argv # Unused. deployment_config = utils.read_yaml_file( utils.normalize_path(FLAGS.deployment_config_path)) output_path = (utils.normalize_path(FLAGS.output_path) if FLAGS.output_path else None) rule_generator.run(deployment_config, output_path=output_path)
def update_generated_fields(input_yaml_path, new_config): """Get and update the generated_fields block of the input yaml.""" cfg_content = utils.read_yaml_file(input_yaml_path) if GENERATED_FIELDS_NAME not in new_config: cfg_content.pop(GENERATED_FIELDS_NAME, {}) else: cfg_content[GENERATED_FIELDS_NAME] = new_config[GENERATED_FIELDS_NAME] return cfg_content
def test_project_config_validate_check_raise(self): FLAGS.projects = ['*'] path = ('deploy/samples/project_with_local_audit_logs.yaml') root_config = utils.read_yaml_file(path) root_config['overall']['allowed_apis'] = [] root_config['projects'][0]['enabled_apis'] = ['foo.googleapis.com'] with tempfile.TemporaryDirectory() as tmp_dir: FLAGS.project_yaml = os.path.join(tmp_dir, 'conf.yaml') with open(FLAGS.project_yaml, 'w') as f: yaml = ruamel.yaml.YAML() yaml.dump(root_config, f) f.flush() with self.assertRaises(utils.InvalidConfigError): create_project.main([])
def test_project_config_validate_check_raise(self): datathon_path = os.path.join(FLAGS.test_srcdir, 'deploy/samples/', 'datathon_team_project.yaml') root_config = utils.resolve_env_vars( utils.read_yaml_file(datathon_path)) root_config['overall']['allowed_apis'] = [] root_config_str = yaml.dump(root_config) with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f: f.write(root_config_str) f.flush() FLAGS.project_yaml = f.name with tempfile.NamedTemporaryFile() as fout: FLAGS.output_yaml_path = fout.name with self.assertRaises(utils.InvalidConfigError): create_project.main([])
def test_project_config_validate_check_raise(self): FLAGS.projects = ['*'] datathon_path = ('deploy/samples/datathon_team_project.yaml') root_config = utils.read_yaml_file(datathon_path) utils.resolve_env_vars(root_config) root_config['overall']['allowed_apis'] = [] with tempfile.TemporaryDirectory() as tmp_dir: FLAGS.output_yaml_path = os.path.join(tmp_dir, 'conf.yaml') FLAGS.project_yaml = FLAGS.output_yaml_path with open(FLAGS.project_yaml, 'w') as f: yaml = ruamel.yaml.YAML() yaml.dump(root_config, f) f.flush() with self.assertRaises(utils.InvalidConfigError): create_project.main([])
def test_project_config_validate_check_correct(self): FLAGS.projects = ['*'] path = ('deploy/samples/project_with_local_audit_logs.yaml') root_config = utils.read_yaml_file(path) root_config['overall']['allowed_apis'] = [ 'bigquery-json.googleapis.com', 'compute.googleapis.com', 'ml.googleapis.com', ] with tempfile.TemporaryDirectory() as tmp_dir: FLAGS.project_yaml = os.path.join(tmp_dir, 'conf.yaml') with open(FLAGS.project_yaml, 'w') as f: yaml = ruamel.yaml.YAML() yaml.dump(root_config, f) f.flush() create_project.main([])
def test_project_config_validate_check_correct(self): datathon_path = os.path.join(FLAGS.test_srcdir, 'deploy/samples/', 'datathon_team_project.yaml') root_config = utils.resolve_env_vars( utils.read_yaml_file(datathon_path)) root_config['overall']['allowed_apis'] = [ 'bigquery-json.googleapis.com', 'compute.googleapis.com', 'ml.googleapis.com', ] root_config_str = yaml.dump(root_config) with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f: f.write(root_config_str) f.flush() FLAGS.project_yaml = f.name with tempfile.NamedTemporaryFile() as fout: FLAGS.output_yaml_path = fout.name create_project.main([])
def test_project_config_validate_check_correct(self): FLAGS.projects = ['*'] datathon_path = ('deploy/samples/datathon_team_project.yaml') root_config = utils.read_yaml_file(datathon_path) utils.resolve_env_vars(root_config) root_config['overall']['allowed_apis'] = [ 'bigquery-json.googleapis.com', 'compute.googleapis.com', 'ml.googleapis.com', ] with tempfile.TemporaryDirectory() as tmp_dir: FLAGS.output_yaml_path = os.path.join(tmp_dir, 'conf.yaml') FLAGS.project_yaml = FLAGS.output_yaml_path with open(FLAGS.project_yaml, 'w') as f: yaml = ruamel.yaml.YAML() yaml.dump(root_config, f) f.flush() create_project.main([])
def main(argv): del argv # Unused. if FLAGS.generated_fields_path == FLAGS.project_yaml: raise Exception( '--generated_fields_path must not be set to the same as --project_yaml.' ) if FLAGS.output_rules_path: FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path) FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml) FLAGS.generated_fields_path = utils.normalize_path( FLAGS.generated_fields_path) # touch file if it has not been created yet open(FLAGS.generated_fields_path, 'a').close() config_string = runner.run_command([ FLAGS.load_config_binary, '--project_yaml_path', FLAGS.project_yaml, '--generated_fields_path', FLAGS.generated_fields_path, ], get_output=True) root_config = yaml.load(config_string) if not root_config: logging.error('Error loading project YAML.') return generated_fields = utils.read_yaml_file(FLAGS.generated_fields_path) if not generated_fields: generated_fields = {'projects': {}} want_projects = set(FLAGS.projects) def want_project(project_config_dict): if not project_config_dict: return False return want_projects == { '*' } or project_config_dict['project_id'] in want_projects projects = [] audit_logs_project = root_config.get('audit_logs_project') # Always deploy the remote audit logs project first (if present). if want_project(audit_logs_project): projects.append( ProjectConfig(root=root_config, project=audit_logs_project, audit_logs_project=None, extra_steps=[], generated_fields=generated_fields)) forseti_config = root_config.get('forseti') if forseti_config and want_project(forseti_config['project']): extra_steps = [ Step( func=install_forseti, description='Install Forseti', updatable=False, ), get_forseti_access_granter_step( forseti_config['project']['project_id']), ] if audit_logs_project: extra_steps.append( get_forseti_access_granter_step( audit_logs_project['project_id'])) forseti_project_config = ProjectConfig( root=root_config, project=forseti_config['project'], audit_logs_project=audit_logs_project, extra_steps=extra_steps, generated_fields=generated_fields) projects.append(forseti_project_config) for project_config in root_config.get('projects', []): if not want_project(project_config): continue extra_steps = [] if forseti_config: extra_steps.append( get_forseti_access_granter_step(project_config['project_id'])) projects.append( ProjectConfig(root=root_config, project=project_config, audit_logs_project=audit_logs_project, extra_steps=extra_steps, generated_fields=generated_fields)) validate_project_configs(root_config['overall'], projects) logging.info('Found %d projects to deploy', len(projects)) for config in projects: logging.info('Setting up project %s', config.project['project_id']) if not setup_project(config): # Don't attempt to deploy additional projects if one project failed. return if forseti_config: call = [ FLAGS.rule_generator_binary, '--project_yaml_path', FLAGS.project_yaml, '--generated_fields_path', FLAGS.generated_fields_path, '--output_path', FLAGS.output_rules_path or '', ] logging.info('Running rule generator: %s', call) utils.call_go_binary(call) logging.info('All projects successfully deployed.')
def main(argv): del argv # Unused. input_yaml_path = utils.normalize_path(FLAGS.project_yaml) output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path) output_rules_path = (utils.normalize_path(FLAGS.output_rules_path) if FLAGS.output_rules_path else None) # Output YAML will rearrange fields and remove comments, so do a basic check # against accidental overwriting. if input_yaml_path == output_yaml_path: logging.error('output_yaml_path cannot overwrite project_yaml.') return # Read and parse the project configuration YAML file. root_config = utils.resolve_env_vars(utils.read_yaml_file(input_yaml_path)) if not root_config: logging.error('Error loading project YAML.') return logging.info('Validating project YAML against schema.') try: utils.validate_config_yaml(root_config) except jsonschema.exceptions.ValidationError as e: logging.error('Error in YAML config: %s', e) return overall = root_config['overall'] audit_logs_project = root_config.get('audit_logs_project') projects = [] # Always deploy the remote audit logs project first (if present). if audit_logs_project: projects.append(ProjectConfig(overall=overall, project=audit_logs_project, audit_logs_project=None)) forseti_config = root_config.get('forseti', {}) if forseti_config: forseti_project_config = ProjectConfig( overall=overall, project=forseti_config.get('project'), audit_logs_project=audit_logs_project) projects.append(forseti_project_config) for project_config in root_config.get('projects', []): projects.append(ProjectConfig(overall=overall, project=project_config, audit_logs_project=audit_logs_project)) validate_project_configs(overall, projects) # If resuming setup from a particular project, skip to that project. if FLAGS.resume_from_project: while (projects and projects[0].project['project_id'] != FLAGS.resume_from_project): skipped = projects.pop(0) logging.info('Skipping project %s', skipped.project['project_id']) if not projects: logging.error('Project not found: %s', FLAGS.resume_from_project) if projects: starting_step = max(1, FLAGS.resume_from_step) for config in projects: logging.info('Setting up project %s', config.project['project_id']) if not setup_new_project(config, starting_step): # Don't attempt to deploy additional projects if one project failed. return # Fill in unset generated fields for the project and save it. add_generated_fields(config.project) utils.write_yaml_file(root_config, output_yaml_path) starting_step = 1 else: logging.error('No projects to deploy.') # TODO: allow for forseti installation to be retried. if forseti_config: forseti_project_id = forseti_config['project']['project_id'] forseti_service_account = forseti_config.get('generated_fields', {}).get('service_account') # If the forseti block does not have generated fields from a previous # deployment, consider Forseti to be undeployed. # TODO: add more checks of a previous deployment. if 'generated_fields' not in forseti_config: forseti.install(forseti_config) forseti_service_account = forseti.get_server_service_account( forseti_project_id) forseti_config['generated_fields'] = { 'service_account': forseti_service_account, 'server_bucket': forseti.get_server_bucket(forseti_project_id), } utils.write_yaml_file(root_config, output_yaml_path) for project in projects: project_id = project.project['project_id'] forseti.grant_access(project_id, forseti_service_account) rule_generator.run(root_config, output_path=output_rules_path)