def deploy_gcs_audit_logs(config): """Deploys the GCS logs bucket to the remote audit logs project, if used.""" # The GCS logs bucket must be created before the data buckets. if not config.audit_logs_project: logging.info('Using local GCS audit logs.') return logs_gcs_bucket = config.project['audit_logs'].get('logs_gcs_bucket') if not logs_gcs_bucket: logging.info('No remote GCS logs bucket required.') return logging.info('Creating remote GCS logs bucket.') data_project_id = config.project['project_id'] logs_project = config.audit_logs_project audit_project_id = logs_project['project_id'] deployment_name = 'audit-logs-{}-gcs'.format( data_project_id.replace('_', '-')) path = os.path.join( os.path.dirname(__file__), 'templates/remote_audit_logs.py') dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [{ 'type': path, 'name': deployment_name, 'properties': { 'owners_group': logs_project['owners_group'], 'auditors_group': config.project['auditors_group'], 'logs_gcs_bucket': logs_gcs_bucket, }, }] } utils.run_deployment(dm_template_dict, deployment_name, audit_project_id)
def create_compute_vms(config): """Creates new GCE VMs and firewall rules if specified in config.""" if 'gce_instances' not in config.project: logging.info('No GCS VMs required.') return project_id = config.project['project_id'] logging.info('Creating GCS VMs.') # Enable OS Login for VM SSH access. runner.run_gcloud_command([ 'compute', 'project-info', 'add-metadata', '--metadata', 'enable-oslogin=TRUE' ], project_id=project_id) gce_instances = [] for instance in config.project['gce_instances']: if 'existing_boot_image' in instance: image_name = instance['existing_boot_image'] else: image_name = ('global/images/' + instance['custom_boot_image']['image_name']) gce_template_dict = { 'name': instance['name'], 'zone': instance['zone'], 'machine_type': instance['machine_type'], 'boot_image_name': image_name, 'start_vm': instance['start_vm'] } startup_script_str = instance.get('startup_script') if startup_script_str: gce_template_dict['metadata'] = { 'items': [{ 'key': 'startup-script', 'value': startup_script_str }] } gce_instances.append(gce_template_dict) deployment_name = 'gce-vms' path = os.path.join(os.path.dirname(__file__), 'templates/gce_vms.py') dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [{ 'type': path, 'name': deployment_name, 'properties': { 'gce_instances': gce_instances, 'firewall_rules': config.project.get('gce_firewall_rules', []), } }] } utils.run_deployment(dm_template_dict, deployment_name, project_id, is_deployed(config.project))
def deploy_bigquery_audit_logs(config): """Deploys the BigQuery audit logs dataset, if used.""" if FLAGS.enable_new_style_resources: logging.info('BQ audit logs will be deployed through CFT.') return data_project_id = config.project['project_id'] logs_dataset = copy.deepcopy( config.project['audit_logs']['logs_bigquery_dataset']) if config.audit_logs_project: logging.info('Creating remote BigQuery logs dataset.') audit_project_id = config.audit_logs_project['project_id'] owners_group = config.audit_logs_project['owners_group'] else: logging.info('Creating local BigQuery logs dataset.') audit_project_id = data_project_id logs_dataset['name'] = 'audit_logs' owners_group = config.project['owners_group'] # Get the service account for the newly-created log sink. logs_dataset[ 'log_sink_service_account'] = utils.get_log_sink_service_account( _LOG_SINK_NAME, data_project_id) deployment_name = 'audit-logs-{}-bq'.format( data_project_id.replace('_', '-')) path = os.path.join(os.path.dirname(__file__), 'templates/remote_audit_logs.py') dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [{ 'type': path, 'name': deployment_name, 'properties': { 'owners_group': owners_group, 'auditors_group': config.project['auditors_group'], 'logs_bigquery_dataset': logs_dataset, }, }] } utils.run_deployment(dm_template_dict, deployment_name, audit_project_id)
def create_compute_vms(config): """Creates new GCE VMs and firewall rules if specified in config.""" if FLAGS.enable_new_style_resources: logging.info('VMs will be deployed through CFT.') return if 'gce_instances' not in config.project: logging.info('No GCS VMs required.') return project_id = config.project['project_id'] logging.info('Creating GCS VMs.') # Enable OS Login for VM SSH access. runner.run_gcloud_command([ 'compute', 'project-info', 'add-metadata', '--metadata', 'enable-oslogin=TRUE' ], project_id=project_id) gce_instances = [] for instance in config.project['gce_instances']: if 'existing_boot_image' in instance: image_name = instance['existing_boot_image'] else: image_name = ('global/images/' + instance['custom_boot_image']['image_name']) gce_template_dict = { 'name': instance['name'], 'zone': instance['zone'], 'machine_type': instance['machine_type'], 'boot_image_name': image_name, 'start_vm': instance['start_vm'] } startup_script_str = instance.get('startup_script') if startup_script_str: gce_template_dict['metadata'] = { 'items': [{ 'key': 'startup-script', 'value': startup_script_str }] } gce_instances.append(gce_template_dict) deployment_name = 'gce-vms' path = os.path.join(os.path.dirname(__file__), 'templates/gce_vms.py') resource = { 'type': path, 'name': deployment_name, 'properties': { 'gce_instances': gce_instances, 'firewall_rules': config.project.get('gce_firewall_rules', []), } } dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [resource], } deployed = utils.deployment_exists(deployment_name, project_id) try: utils.run_deployment(dm_template_dict, deployment_name, project_id) except subprocess.CalledProcessError: # Only retry vm deployment for updates if not deployed: raise # TODO: check error message for whether failure was specifically # due to vm running logging.info(('Potential failure due to updating a running vm. ' 'Retrying with vm shutdown.')) vm_names_to_shutdown = [ instance['name'] for instance in config.project.get('gce_instances', []) ] resource['properties']['vm_names_to_shutdown'] = vm_names_to_shutdown utils.run_deployment(dm_template_dict, deployment_name, project_id)
def deploy_project_resources(config): """Deploys resources into the new data project.""" if FLAGS.enable_new_style_resources: logging.info('Project resources will be deployed through CFT.') return setup_account = utils.get_gcloud_user() has_organization = bool(config.root['overall'].get('organization_id')) project_id = config.project['project_id'] dm_service_account = utils.get_deployment_manager_service_account(project_id) # Build a deployment config for the data_project.py deployment manager # template. properties = copy.deepcopy(config.project) # Remove the current user as an owner of the project if project is part of an # organization. properties['has_organization'] = has_organization if has_organization: properties['remove_owner_user'] = setup_account # Change audit_logs to either local_audit_logs or remote_audit_logs in the # deployment manager template properties. audit_logs = properties.pop('audit_logs') if config.audit_logs_project: properties['remote_audit_logs'] = { 'audit_logs_project_id': config.audit_logs_project['project_id'], 'logs_bigquery_dataset_id': audit_logs['logs_bigquery_dataset']['name'], } # Logs GCS bucket is not required for projects without data GCS buckets. if 'logs_gcs_bucket' in audit_logs: properties['remote_audit_logs']['logs_gcs_bucket_name'] = ( audit_logs['logs_gcs_bucket']['name']) else: properties['local_audit_logs'] = audit_logs # Set data buckets' names. for data_bucket in properties.get('data_buckets', []): data_bucket['name'] = get_data_bucket_name(data_bucket, project_id) data_bucket.pop('name_suffix', '') path = os.path.join(os.path.dirname(__file__), 'templates/data_project.py') dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [{ 'type': path, 'name': 'data_project_deployment', 'properties': properties, }] } # API iam.googleapis.com is necessary when using custom roles iam_api_disable = False if not _is_service_enabled('iam.googleapis.com', project_id): runner.run_gcloud_command(['services', 'enable', 'iam.googleapis.com'], project_id=project_id) iam_api_disable = True try: # Create the deployment. utils.run_deployment(dm_template_dict, 'data-project-deployment', project_id) for role in _DEPLOYMENT_MANAGER_ROLES: runner.run_gcloud_command([ 'projects', 'remove-iam-policy-binding', project_id, '--member', dm_service_account, '--role', role ], project_id=None) finally: # Disable iam.googleapis.com if it is enabled in this function if iam_api_disable: runner.run_gcloud_command(['services', 'disable', 'iam.googleapis.com'], project_id=project_id)
def deploy_project_resources(config): """Deploys resources into the new data project.""" setup_account = utils.get_gcloud_user() has_organization = bool(config.root['overall'].get('organization_id')) project_id = config.project['project_id'] dm_service_account = utils.get_deployment_manager_service_account(project_id) # Build a deployment config for the data_project.py deployment manager # template. properties = copy.deepcopy(config.project) # Remove the current user as an owner of the project if project is part of an # organization. properties['has_organization'] = has_organization if has_organization: properties['remove_owner_user'] = setup_account # Change audit_logs to either local_audit_logs or remote_audit_logs in the # deployment manager template properties. audit_logs = properties.pop('audit_logs') if config.audit_logs_project: properties['remote_audit_logs'] = { 'audit_logs_project_id': config.audit_logs_project['project_id'], 'logs_bigquery_dataset_id': audit_logs['logs_bigquery_dataset']['name'], } # Logs GCS bucket is not required for projects without data GCS buckets. if 'logs_gcs_bucket' in audit_logs: properties['remote_audit_logs']['logs_gcs_bucket_name'] = ( audit_logs['logs_gcs_bucket']['name']) else: properties['local_audit_logs'] = audit_logs path = os.path.join(os.path.dirname(__file__), 'templates/data_project.py') dm_template_dict = { 'imports': [{ 'path': path }], 'resources': [{ 'type': path, 'name': 'data_project_deployment', 'properties': properties, }] } # API iam.googleapis.com is necessary when using custom roles iam_api_disable = False if not _is_service_enabled('iam.googleapis.com', project_id): runner.run_gcloud_command(['services', 'enable', 'iam.googleapis.com'], project_id=project_id) iam_api_disable = True try: # Create the deployment. utils.run_deployment(dm_template_dict, 'data-project-deployment', project_id) # Create project liens if requested. if config.project.get('create_deletion_lien'): existing_restrictions = runner.run_gcloud_command( [ 'alpha', 'resource-manager', 'liens', 'list', '--format', 'value(restrictions)' ], project_id=project_id).split('\n') if _LIEN_RESTRICTION not in existing_restrictions: runner.run_gcloud_command([ 'alpha', 'resource-manager', 'liens', 'create', '--restrictions', _LIEN_RESTRICTION, '--reason', 'Automated project deletion lien deployment.' ], project_id=project_id) for role in _DEPLOYMENT_MANAGER_ROLES: runner.run_gcloud_command([ 'projects', 'remove-iam-policy-binding', project_id, '--member', dm_service_account, '--role', role ], project_id=None) finally: # Disable iam.googleapis.com if it is enabled in this function if iam_api_disable: runner.run_gcloud_command(['services', 'disable', 'iam.googleapis.com'], project_id=project_id)