def CreateComputeImages(config):
"""Creates new Compute Engine VM images if specified in config."""
gce_instances = config.project.get('gce_instances')
if not gce_instances:
logging.info('No GCS Images required.')
return
project_id = config.project['project_id']
for instance in gce_instances:
custom_image = instance.get('custom_boot_image')
if not custom_image:
logging.info('Using existing compute image %s.',
instance['existing_boot_image'])
continue
# Check if custom image already exists.
if utils.RunGcloudCommand([
'compute', 'images', 'list', '--no-standard-images',
'--filter', 'name={}'.format(
custom_image['image_name']), '--format', 'value(name)'
],
project_id=project_id):
logging.info('Image %s already exists, skipping image creation.',
custom_image['image_name'])
continue
logging.info('Creating VM Image %s.', custom_image['image_name'])
# Create VM image using gcloud rather than deployment manager so that the
# deployment manager service account doesn't need to be granted access to
# the image GCS bucket.
image_uri = 'gs://' + custom_image['gcs_path']
utils.RunGcloudCommand([
'compute', 'images', 'create', custom_image['image_name'],
'--source-uri', image_uri
],
project_id=project_id)
def EnableDeploymentManager(config):
"""Enables Deployment manager, with role/owners for its service account."""
logging.info('Setting up Deployment Manager...')
project_id = config['project_config']['project_id']
# Enabled Deployment Manger and Cloud Resource Manager for this project.
utils.RunGcloudCommand([
'services', 'enable', 'deploymentmanager',
'cloudresourcemanager.googleapis.com'
], project_id)
# Grant deployment manager service account (temporary) owners access.
dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id)
utils.RunGcloudCommand([
'projects', 'add-iam-policy-binding', project_id, '--member',
dm_service_account, '--role', 'roles/owner'
],
project_id=None)
def DeployProjectResources(config):
"""Deploys resources into the new data project."""
logging.info('Deploying Project resources...')
setup_account = utils.GetGcloudUser()
has_organization = bool(config.overall.get('organization_id'))
project_id = config.project['project_id']
dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id)
# Build a deployment config for the data_project.py deployment manager
# template.
# Shallow copy is sufficient for this script.
properties = config.project.copy()
# Remove the current user as an owner of the project if project is part of an
# organization.
properties['has_organization'] = has_organization
if has_organization:
properties['remove_owner_user'] = setup_account
# Change audit_logs to either local_audit_logs or remote_audit_logs in the
# deployment manager template properties.
audit_logs = properties.pop('audit_logs')
if config.audit_logs_project:
properties['remote_audit_logs'] = {
'audit_logs_project_id':
config.audit_logs_project['project_id'],
'logs_bigquery_dataset_id':
audit_logs['logs_bigquery_dataset']['name'],
}
# Logs GCS bucket is not required for projects without data GCS buckets.
if 'logs_gcs_bucket' in audit_logs:
properties['remote_audit_logs']['logs_gcs_bucket_name'] = (
audit_logs['logs_gcs_bucket']['name'])
else:
properties['local_audit_logs'] = audit_logs
dm_template_dict = {
'imports': [{
'path': 'data_project.py'
}],
'resources': [{
'type': 'data_project.py',
'name': 'data_project_deployment',
'properties': properties,
}]
}
# Create the deployment.
utils.CreateNewDeployment(dm_template_dict, 'data-project-deployment',
project_id)
# Remove Owners role from the DM service account.
utils.RunGcloudCommand([
'projects', 'remove-iam-policy-binding', project_id, '--member',
dm_service_account, '--role', 'roles/owner'
],
project_id=None)
def SetupBilling(config):
"""Sets the billing account for this project."""
logging.info('Setting up billing...')
billing_acct = config['billing_account']
project_id = config['project_config']['project_id']
# Set the appropriate billing account for this project:
utils.RunGcloudCommand([
'beta', 'billing', 'projects', 'link', project_id, '--billing-account',
billing_acct
],
project_id=None)
def CreateNewProject(config):
"""Creates the new GCP project."""
logging.info('Creating a new GCP project...')
org_id = config.get('organization_id')
project_id = config['project_config']['project_id']
create_project_command = ['projects', 'create', project_id]
if org_id:
create_project_command.extend(['--organization', str(org_id)])
else:
logging.info('Deploying without a parent organization.')
# Create the new project.
utils.RunGcloudCommand(create_project_command, project_id=None)
def DeployProjectResources(config):
"""Deploys resources into the new data project."""
logging.info('Deploying Data Project resources...')
setup_account = utils.GetGcloudUser()
has_organization = bool(config.get('organization_id'))
project_id = config['project_config']['project_id']
dm_service_account = utils.GetDeploymentManagerServiceAccount(project_id)
# Build a deployment config for the data_project.py deployment manager
# template.
# Shallow copy is sufficient for this script.
properties = config['project_config'].copy()
# Remove the current user as an owner of the project if project is part of an
# organization.
properties['has_organization'] = has_organization
if has_organization:
properties['remove_owner_user'] = setup_account
# If using remote_audit_logs, set properties for the data project.
remote_audit_logs = config.get('remote_audit_logs')
if remote_audit_logs:
properties['remote_audit_logs'] = {
'audit_logs_project_id':
remote_audit_logs['audit_logs_project_id'],
'logs_gcs_bucket_name':
remote_audit_logs['logs_gcs_bucket']['name'],
'logs_bigquery_dataset_id':
(remote_audit_logs['logs_bigquery_dataset']['name']),
}
dm_template_dict = {
'imports': [{
'path': 'data_project.py'
}],
'resources': [{
'type': 'data_project.py',
'name': 'data_project_deployment',
'properties': properties,
}]
}
# Create the deployment.
utils.CreateNewDeployment(dm_template_dict, 'data-project-deployment',
project_id)
# Remove Owners role from the DM service account.
utils.RunGcloudCommand([
'projects', 'remove-iam-policy-binding', project_id, '--member',
dm_service_account, '--role', 'roles/owner'
],
project_id=None)
def CreateStackdriverAccount(config):
"""Prompts the user to create a new Stackdriver Account."""
# Creating a Stackdriver account cannot be done automatically, so ask the
# user to create one.
if 'stackdriver_alert_email' not in config['project_config']:
logging.warning(
'No Stackdriver alert email specified, skipping creation '
'of Stackdriver account.')
return
logging.info('Creating Stackdriver account.')
project_id = config['project_config']['project_id']
message = """
------------------------------------------------------------------------------
To create email alerts, this project needs a Stackdriver account.
Create a new Stackdriver account for this project by visiting:
https://console.cloud.google.com/monitoring?project={}
Only add this project, and skip steps for adding additional GCP or AWS
projects. You don't need to install Stackdriver Agents.
IMPORTANT: Wait about 5 minutes for the account to be created.
For more information, see: https://cloud.google.com/monitoring/accounts/
After the account is created, enter [Y] to continue, or enter [N] to skip the
creation of Stackdriver alerts.
------------------------------------------------------------------------------
""".format(project_id)
print(message)
# Keep trying until Stackdriver account is ready, or user skips.
while True:
if not utils.WaitForYesNo('Account created [y/N]?'):
logging.warning('Skipping creation of Stackdriver Account.')
return
# Verify account was created.
try:
utils.RunGcloudCommand(['alpha', 'monitoring', 'policies', 'list'],
project_id)
return
except utils.GcloudRuntimeError as e:
logging.error('Error reading Stackdriver account %s', e)
print('Could not find Stackdriver account.')
def CreateComputeVms(config):
"""Creates new GCE VMs and firewall rules if specified in config."""
if 'gce_instances' not in config.project:
logging.info('No GCS VMs required.')
return
project_id = config.project['project_id']
logging.info('Creating GCS VMs.')
# Enable OS Login for VM SSH access.
utils.RunGcloudCommand([
'compute', 'project-info', 'add-metadata', '--metadata',
'enable-oslogin=TRUE'
],
project_id=project_id)
gce_instances = []
for instance in config.project['gce_instances']:
if 'existing_boot_image' in instance:
image_name = instance['existing_boot_image']
else:
image_name = ('global/images/' +
instance['custom_boot_image']['image_name'])
gce_instances.append({
'name': instance['name'],
'zone': instance['zone'],
'machine_type': instance['machine_type'],
'boot_image_name': image_name,
'start_vm': instance['start_vm'],
})
deployment_name = 'gce-vms'
dm_template_dict = {
'imports': [{
'path': 'gce_vms.py'
}],
'resources': [{
'type': 'gce_vms.py',
'name': deployment_name,
'properties': {
'gce_instances': gce_instances,
'firewall_rules': config.project.get('gce_firewall_rules', []),
}
}]
}
utils.CreateNewDeployment(dm_template_dict, deployment_name, project_id)
