def test_project_config_validate_check_raise(self):
FLAGS.projects = ['*']
datathon_path = ('deploy/samples/datathon_team_project.yaml')
root_config = utils.read_yaml_file(datathon_path)
utils.resolve_env_vars(root_config)
root_config['overall']['allowed_apis'] = []
with tempfile.TemporaryDirectory() as tmp_dir:
FLAGS.output_yaml_path = os.path.join(tmp_dir, 'conf.yaml')
FLAGS.project_yaml = FLAGS.output_yaml_path
with open(FLAGS.project_yaml, 'w') as f:
yaml = ruamel.yaml.YAML()
yaml.dump(root_config, f)
f.flush()
with self.assertRaises(utils.InvalidConfigError):
create_project.main([])
def test_project_config_validate_check_correct(self):
FLAGS.projects = ['*']
datathon_path = ('deploy/samples/datathon_team_project.yaml')
root_config = utils.read_yaml_file(datathon_path)
utils.resolve_env_vars(root_config)
root_config['overall']['allowed_apis'] = [
'bigquery-json.googleapis.com',
'compute.googleapis.com',
'ml.googleapis.com',
]
with tempfile.TemporaryDirectory() as tmp_dir:
FLAGS.output_yaml_path = os.path.join(tmp_dir, 'conf.yaml')
FLAGS.project_yaml = FLAGS.output_yaml_path
with open(FLAGS.project_yaml, 'w') as f:
yaml = ruamel.yaml.YAML()
yaml.dump(root_config, f)
f.flush()
create_project.main([])
def test_project_config_validate_check_raise(self):
datathon_path = os.path.join(FLAGS.test_srcdir, 'deploy/samples/',
'datathon_team_project.yaml')
root_config = utils.resolve_env_vars(
utils.read_yaml_file(datathon_path))
root_config['overall']['allowed_apis'] = []
root_config_str = yaml.dump(root_config)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
f.write(root_config_str)
f.flush()
FLAGS.project_yaml = f.name
with tempfile.NamedTemporaryFile() as fout:
FLAGS.output_yaml_path = fout.name
with self.assertRaises(utils.InvalidConfigError):
create_project.main([])
def test_project_config_validate_check_correct(self):
datathon_path = os.path.join(FLAGS.test_srcdir, 'deploy/samples/',
'datathon_team_project.yaml')
root_config = utils.resolve_env_vars(
utils.read_yaml_file(datathon_path))
root_config['overall']['allowed_apis'] = [
'bigquery-json.googleapis.com',
'compute.googleapis.com',
'ml.googleapis.com',
]
root_config_str = yaml.dump(root_config)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
f.write(root_config_str)
f.flush()
FLAGS.project_yaml = f.name
with tempfile.NamedTemporaryFile() as fout:
FLAGS.output_yaml_path = fout.name
create_project.main([])
def main(argv):
del argv # Unused.
input_yaml_path = utils.normalize_path(FLAGS.project_yaml)
output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
output_rules_path = (utils.normalize_path(FLAGS.output_rules_path)
if FLAGS.output_rules_path else None)
# Output YAML will rearrange fields and remove comments, so do a basic check
# against accidental overwriting.
if input_yaml_path == output_yaml_path:
logging.error('output_yaml_path cannot overwrite project_yaml.')
return
# Read and parse the project configuration YAML file.
root_config = utils.resolve_env_vars(utils.read_yaml_file(input_yaml_path))
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
overall = root_config['overall']
audit_logs_project = root_config.get('audit_logs_project')
projects = []
# Always deploy the remote audit logs project first (if present).
if audit_logs_project:
projects.append(ProjectConfig(overall=overall,
project=audit_logs_project,
audit_logs_project=None))
forseti_config = root_config.get('forseti', {})
if forseti_config:
forseti_project_config = ProjectConfig(
overall=overall,
project=forseti_config.get('project'),
audit_logs_project=audit_logs_project)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
projects.append(ProjectConfig(overall=overall,
project=project_config,
audit_logs_project=audit_logs_project))
validate_project_configs(overall, projects)
# If resuming setup from a particular project, skip to that project.
if FLAGS.resume_from_project:
while (projects and
projects[0].project['project_id'] != FLAGS.resume_from_project):
skipped = projects.pop(0)
logging.info('Skipping project %s', skipped.project['project_id'])
if not projects:
logging.error('Project not found: %s', FLAGS.resume_from_project)
if projects:
starting_step = max(1, FLAGS.resume_from_step)
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_new_project(config, starting_step):
# Don't attempt to deploy additional projects if one project failed.
return
# Fill in unset generated fields for the project and save it.
add_generated_fields(config.project)
utils.write_yaml_file(root_config, output_yaml_path)
starting_step = 1
else:
logging.error('No projects to deploy.')
# TODO: allow for forseti installation to be retried.
if forseti_config:
forseti_project_id = forseti_config['project']['project_id']
forseti_service_account = forseti_config.get('generated_fields',
{}).get('service_account')
# If the forseti block does not have generated fields from a previous
# deployment, consider Forseti to be undeployed.
# TODO: add more checks of a previous deployment.
if 'generated_fields' not in forseti_config:
forseti.install(forseti_config)
forseti_service_account = forseti.get_server_service_account(
forseti_project_id)
forseti_config['generated_fields'] = {
'service_account': forseti_service_account,
'server_bucket': forseti.get_server_bucket(forseti_project_id),
}
utils.write_yaml_file(root_config, output_yaml_path)
for project in projects:
project_id = project.project['project_id']
forseti.grant_access(project_id,
forseti_service_account)
rule_generator.run(root_config, output_path=output_rules_path)