def install(config):
"""Install a Forseti instance in the given project config.
Args:
config (dict): Forseti config dict of the Forseti instance to deploy.
"""
tmp_dir = tempfile.mkdtemp()
try:
# clone repo
runner.run_command(['git', 'clone', _FORSETI_REPO, tmp_dir])
# make sure we're running from the default branch
runner.run_command(['git', '-C', tmp_dir, 'checkout', _DEFAULT_BRANCH])
# TODO: Pass in a project_id flag once
# https://github.com/GoogleCloudPlatform/forseti-security/issues/2182
# is closed.
runner.run_command([
'gcloud', 'config', 'set', 'project', config['project']['project_id'],
])
# run forseti installer
install_cmd = [
'python', os.path.join(tmp_dir, 'install/gcp_installer.py'),
'--no-cloudshell',
]
if 'installer_flags' in config:
install_cmd.extend(shlex.split(config['installer_flags']))
runner.run_command(install_cmd)
finally:
shutil.rmtree(tmp_dir)
def run(deployment_config, output_path=None):
"""Run the rule generator.
Generate rules for all supported scanners based on the given deployment config
and write them in the given output directory.
Args:
deployment_config(dict): The loaded yaml deployment config.
output_path (str): Path to a local directory or a GCS bucket
path starting with gs://.
Raises:
ValueError: If no output_path given AND no forseti config in the
deployment_config.
"""
if not output_path:
output_path = deployment_config.get('forseti',
{}).get('generated_fields',
{}).get('server_bucket')
if not output_path:
raise ValueError((
'Must provide an output path or set the "forseti_server_bucket" '
'field in the overall generated_fields'))
if output_path.startswith('gs://'):
# output path is a GCS bucket
with tempfile.TemporaryDirectory() as tmp_dir:
_write_rules(deployment_config, tmp_dir)
logging.info('Copying rules files to %s', output_path)
runner.run_command([
'gsutil',
'cp',
os.path.join(tmp_dir, '*.yaml'),
posixpath.join(output_path, 'rules'),
])
else:
# output path is a local directory
_write_rules(deployment_config, output_path)
def get_server_bucket(forseti_project_id):
"""Get the bucket holding the Forseti server instance's configuration.
Args:
forseti_project_id (str): id of the Forseti project.
Returns:
str: the forseti server bucket name.
Raises:
ValueError: if failure in finding the bucket in the gsutil command output.
"""
output = runner.run_command(['gsutil', 'ls', '-p', forseti_project_id],
get_output=True)
match = _FORSETI_SERVER_BUCKET_RE.search(output)
if not match:
raise ValueError(
'Failed to find Forseti server bucket: {}'.format(output))
return match.group(0)
def get_server_bucket(forseti_project_id):
"""Get the bucket holding the Forseti server instance's configuration.
Args:
forseti_project_id (str): id of the Forseti project.
Returns:
str: the forseti server bucket name.
Raises:
ValueError: if failure in finding the bucket in the gsutil command output.
"""
# TODO: allow users to override gsutil binary, similar to gcloud
output = runner.run_command(['gsutil', 'ls', '-p', forseti_project_id],
get_output=True)
if FLAGS.dry_run:
return '__DRY_RUN_FORSETI_BUCKET__'
match = _FORSETI_SERVER_BUCKET_RE.search(output)
if not match:
raise ValueError(
'Failed to find Forseti server bucket: {}'.format(output))
return match.group(0)
def main(argv):
del argv # Unused.
if FLAGS.generated_fields_path == FLAGS.project_yaml:
raise Exception(
'--generated_fields_path must not be set to the same as --project_yaml.'
)
if FLAGS.output_rules_path:
FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml)
FLAGS.generated_fields_path = utils.normalize_path(
FLAGS.generated_fields_path)
# touch file if it has not been created yet
open(FLAGS.generated_fields_path, 'a').close()
config_string = runner.run_command([
FLAGS.load_config_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--generated_fields_path',
FLAGS.generated_fields_path,
],
get_output=True)
root_config = yaml.load(config_string)
if not root_config:
logging.error('Error loading project YAML.')
return
generated_fields = utils.read_yaml_file(FLAGS.generated_fields_path)
if not generated_fields:
generated_fields = {'projects': {}}
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[],
generated_fields=generated_fields))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(
audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps,
generated_fields=generated_fields)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps,
generated_fields=generated_fields))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--generated_fields_path',
FLAGS.generated_fields_path,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
utils.call_go_binary(call)
logging.info('All projects successfully deployed.')
def main(argv):
del argv # Unused.
if FLAGS.enable_new_style_resources:
logging.info('--enable_new_style_resources is true.')
FLAGS.output_yaml_path = utils.normalize_path(FLAGS.output_yaml_path)
if FLAGS.output_rules_path:
FLAGS.output_rules_path = utils.normalize_path(FLAGS.output_rules_path)
FLAGS.project_yaml = utils.normalize_path(FLAGS.project_yaml)
if FLAGS.enable_new_style_resources:
config_string = runner.run_command([
FLAGS.load_config_binary,
'--config_path',
FLAGS.project_yaml,
],
get_output=True)
yaml = ruamel.yaml.YAML()
root_config = yaml.load(config_string)
else:
root_config = utils.load_config(FLAGS.project_yaml)
if not root_config:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(root_config)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
want_projects = set(FLAGS.projects)
def want_project(project_config_dict):
if not project_config_dict:
return False
return want_projects == {
'*'
} or project_config_dict['project_id'] in want_projects
projects = []
audit_logs_project = root_config.get('audit_logs_project')
# Always deploy the remote audit logs project first (if present).
if want_project(audit_logs_project):
projects.append(
ProjectConfig(
root=root_config,
project=audit_logs_project,
audit_logs_project=None,
extra_steps=[]))
forseti_config = root_config.get('forseti')
if forseti_config and want_project(forseti_config['project']):
extra_steps = [
Step(
func=install_forseti,
description='Install Forseti',
updatable=False,
),
get_forseti_access_granter_step(
forseti_config['project']['project_id']),
]
if audit_logs_project:
extra_steps.append(
get_forseti_access_granter_step(audit_logs_project['project_id']))
forseti_project_config = ProjectConfig(
root=root_config,
project=forseti_config['project'],
audit_logs_project=audit_logs_project,
extra_steps=extra_steps)
projects.append(forseti_project_config)
for project_config in root_config.get('projects', []):
if not want_project(project_config):
continue
extra_steps = []
if forseti_config:
extra_steps.append(
get_forseti_access_granter_step(project_config['project_id']))
projects.append(
ProjectConfig(
root=root_config,
project=project_config,
audit_logs_project=audit_logs_project,
extra_steps=extra_steps))
validate_project_configs(root_config['overall'], projects)
logging.info('Found %d projects to deploy', len(projects))
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config, FLAGS.project_yaml, FLAGS.output_yaml_path):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
if FLAGS.enable_new_style_resources:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
utils.call_go_binary(call)
else:
rule_generator.run(root_config, output_path=FLAGS.output_rules_path)
logging.info(
'All projects successfully deployed. Please remember to sync '
'any changes written to the config at --output_yaml_path with '
'--project_yaml before running the script again (Note: only applicable '
'if --output_yaml_path != --project_yaml)')