def CreateComputeImages(config): """Creates new Compute Engine VM images if specified in config.""" gce_instances = config.project.get('gce_instances') if not gce_instances: logging.info('No GCS Images required.') return project_id = config.project['project_id'] for instance in gce_instances: custom_image = instance.get('custom_boot_image') if not custom_image: logging.info('Using existing compute image %s.', instance['existing_boot_image']) continue # Check if custom image already exists. if utils.RunGcloudCommand([ 'compute', 'images', 'list', '--no-standard-images', '--filter', 'name={}'.format( custom_image['image_name']), '--format', 'value(name)' ], project_id=project_id): logging.info('Image %s already exists, skipping image creation.', custom_image['image_name']) continue logging.info('Creating VM Image %s.', custom_image['image_name']) # Create VM image using gcloud rather than deployment manager so that the # deployment manager service account doesn't need to be granted access to # the image GCS bucket. image_uri = 'gs://' + custom_image['gcs_path'] utils.RunGcloudCommand([ 'compute', 'images', 'create', custom_image['image_name'], '--source-uri', image_uri ], project_id=project_id)
def EnableDeploymentManager(config): """Enables Deployment manager, with role/owners for its service account.""" logging.info('Setting up Deployment Manager...') project_id = config['project_config']['project_id'] # Enabled Deployment Manger and Cloud Resource Manager for this project. utils.RunGcloudCommand([ 'services', 'enable', 'deploymentmanager', 'cloudresourcemanager.googleapis.com' ], project_id) # Grant deployment manager service account (temporary) owners access. dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id) utils.RunGcloudCommand([ 'projects', 'add-iam-policy-binding', project_id, '--member', dm_service_account, '--role', 'roles/owner' ], project_id=None)
def DeployProjectResources(config): """Deploys resources into the new data project.""" logging.info('Deploying Project resources...') setup_account = utils.GetGcloudUser() has_organization = bool(config.overall.get('organization_id')) project_id = config.project['project_id'] dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id) # Build a deployment config for the data_project.py deployment manager # template. # Shallow copy is sufficient for this script. properties = config.project.copy() # Remove the current user as an owner of the project if project is part of an # organization. properties['has_organization'] = has_organization if has_organization: properties['remove_owner_user'] = setup_account # Change audit_logs to either local_audit_logs or remote_audit_logs in the # deployment manager template properties. audit_logs = properties.pop('audit_logs') if config.audit_logs_project: properties['remote_audit_logs'] = { 'audit_logs_project_id': config.audit_logs_project['project_id'], 'logs_bigquery_dataset_id': audit_logs['logs_bigquery_dataset']['name'], } # Logs GCS bucket is not required for projects without data GCS buckets. if 'logs_gcs_bucket' in audit_logs: properties['remote_audit_logs']['logs_gcs_bucket_name'] = ( audit_logs['logs_gcs_bucket']['name']) else: properties['local_audit_logs'] = audit_logs dm_template_dict = { 'imports': [{ 'path': 'data_project.py' }], 'resources': [{ 'type': 'data_project.py', 'name': 'data_project_deployment', 'properties': properties, }] } # Create the deployment. utils.CreateNewDeployment(dm_template_dict, 'data-project-deployment', project_id) # Remove Owners role from the DM service account. utils.RunGcloudCommand([ 'projects', 'remove-iam-policy-binding', project_id, '--member', dm_service_account, '--role', 'roles/owner' ], project_id=None)
def SetupBilling(config): """Sets the billing account for this project.""" logging.info('Setting up billing...') billing_acct = config['billing_account'] project_id = config['project_config']['project_id'] # Set the appropriate billing account for this project: utils.RunGcloudCommand([ 'beta', 'billing', 'projects', 'link', project_id, '--billing-account', billing_acct ], project_id=None)
def CreateNewProject(config): """Creates the new GCP project.""" logging.info('Creating a new GCP project...') org_id = config.get('organization_id') project_id = config['project_config']['project_id'] create_project_command = ['projects', 'create', project_id] if org_id: create_project_command.extend(['--organization', str(org_id)]) else: logging.info('Deploying without a parent organization.') # Create the new project. utils.RunGcloudCommand(create_project_command, project_id=None)
def DeployProjectResources(config): """Deploys resources into the new data project.""" logging.info('Deploying Data Project resources...') setup_account = utils.GetGcloudUser() has_organization = bool(config.get('organization_id')) project_id = config['project_config']['project_id'] dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id) # Build a deployment config for the data_project.py deployment manager # template. # Shallow copy is sufficient for this script. properties = config['project_config'].copy() # Remove the current user as an owner of the project if project is part of an # organization. properties['has_organization'] = has_organization if has_organization: properties['remove_owner_user'] = setup_account # If using remote_audit_logs, set properties for the data project. remote_audit_logs = config.get('remote_audit_logs') if remote_audit_logs: properties['remote_audit_logs'] = { 'audit_logs_project_id': remote_audit_logs['audit_logs_project_id'], 'logs_gcs_bucket_name': remote_audit_logs['logs_gcs_bucket']['name'], 'logs_bigquery_dataset_id': (remote_audit_logs['logs_bigquery_dataset']['name']), } dm_template_dict = { 'imports': [{ 'path': 'data_project.py' }], 'resources': [{ 'type': 'data_project.py', 'name': 'data_project_deployment', 'properties': properties, }] } # Create the deployment. utils.CreateNewDeployment(dm_template_dict, 'data-project-deployment', project_id) # Remove Owners role from the DM service account. utils.RunGcloudCommand([ 'projects', 'remove-iam-policy-binding', project_id, '--member', dm_service_account, '--role', 'roles/owner' ], project_id=None)
def CreateStackdriverAccount(config): """Prompts the user to create a new Stackdriver Account.""" # Creating a Stackdriver account cannot be done automatically, so ask the # user to create one. if 'stackdriver_alert_email' not in config['project_config']: logging.warning( 'No Stackdriver alert email specified, skipping creation ' 'of Stackdriver account.') return logging.info('Creating Stackdriver account.') project_id = config['project_config']['project_id'] message = """ ------------------------------------------------------------------------------ To create email alerts, this project needs a Stackdriver account. Create a new Stackdriver account for this project by visiting: https://console.cloud.google.com/monitoring?project={} Only add this project, and skip steps for adding additional GCP or AWS projects. You don't need to install Stackdriver Agents. IMPORTANT: Wait about 5 minutes for the account to be created. For more information, see: https://cloud.google.com/monitoring/accounts/ After the account is created, enter [Y] to continue, or enter [N] to skip the creation of Stackdriver alerts. ------------------------------------------------------------------------------ """.format(project_id) print(message) # Keep trying until Stackdriver account is ready, or user skips. while True: if not utils.WaitForYesNo('Account created [y/N]?'): logging.warning('Skipping creation of Stackdriver Account.') return # Verify account was created. try: utils.RunGcloudCommand(['alpha', 'monitoring', 'policies', 'list'], project_id) return except utils.GcloudRuntimeError as e: logging.error('Error reading Stackdriver account %s', e) print('Could not find Stackdriver account.')
def CreateComputeVms(config): """Creates new GCE VMs and firewall rules if specified in config.""" if 'gce_instances' not in config.project: logging.info('No GCS VMs required.') return project_id = config.project['project_id'] logging.info('Creating GCS VMs.') # Enable OS Login for VM SSH access. utils.RunGcloudCommand([ 'compute', 'project-info', 'add-metadata', '--metadata', 'enable-oslogin=TRUE' ], project_id=project_id) gce_instances = [] for instance in config.project['gce_instances']: if 'existing_boot_image' in instance: image_name = instance['existing_boot_image'] else: image_name = ('global/images/' + instance['custom_boot_image']['image_name']) gce_instances.append({ 'name': instance['name'], 'zone': instance['zone'], 'machine_type': instance['machine_type'], 'boot_image_name': image_name, 'start_vm': instance['start_vm'], }) deployment_name = 'gce-vms' dm_template_dict = { 'imports': [{ 'path': 'gce_vms.py' }], 'resources': [{ 'type': 'gce_vms.py', 'name': deployment_name, 'properties': { 'gce_instances': gce_instances, 'firewall_rules': config.project.get('gce_firewall_rules', []), } }] } utils.CreateNewDeployment(dm_template_dict, deployment_name, project_id)