def SetDiskProperties(context, disks, add_blank_src_img=False):
"""Set properties on each disk to required format.
Sets default values, and moves properties passed directly into
initializeParams where required.
Args:
context: Template context dictionary.
disks: List of disks to set properties on.
add_blank_src_img: When true, link to blank source image is added for new
disks where a source image is not specified.
"""
project = context.env[default.PROJECT]
zone = context.properties.setdefault(ZONE, DEFAULT_ZONE)
for disk in disks:
disk.setdefault(default.AUTO_DELETE, True)
disk.setdefault('boot', False)
disk.setdefault(default.TYPE, DEFAULT_PERSISTENT)
# If disk already exists, no properties to change.
if default.DISK_SOURCE in disk:
continue
else:
disk_init = disk.setdefault(default.INITIALIZEP, dict())
if disk[default.TYPE] == SCRATCH:
disk_init.setdefault(DISKTYPE, 'local-ssd')
else:
# In the Instance API reference, size and type are within this property
if disk_init:
disk_init.setdefault(default.DISK_SIZE,
DEFAULT_DATADISKSIZE)
disk_init.setdefault(default.DISKTYPE, DEFAULT_DISKTYPE)
# You can also simply pass the size and type properties directly
else:
disk_init[default.DISK_SIZE] = disk.pop(
default.DISK_SIZE, DEFAULT_DATADISKSIZE)
disk_init[default.DISKTYPE] = disk.pop(
default.DISKTYPE, DEFAULT_DISKTYPE)
# If disk name was given as a direct property, move to initializeParams
if default.DISK_NAME in disk:
disk_init[default.DISK_NAME] = disk.pop(default.DISK_NAME)
# Add link to a blank source image where non-specified
if add_blank_src_img and default.SRCIMAGE not in disk_init:
disk_init[default.SRCIMAGE] = common.MakeC2DImageLink(
BLANK_IMAGE)
# Change disk type names into URLs
disk_init[default.DISKTYPE] = common.LocalComputeLink(
project, zone, 'diskTypes', disk_init[default.DISKTYPE])
def GenerateComputeVM(context, create_disks_separately=True):
"""Generates one VM instance resource.
Args:
context: Template context dictionary.
create_disks_separately: When true (default), all new disk resources are
created as separate resources. This is legacy behaviour from when multiple
disks creation was not allowed in the disks property.
Returns:
dictionary representing instance resource.
"""
prop = context.properties
boot_disk_type = prop.setdefault(BOOTDISKTYPE, DEFAULT_DISKTYPE)
prop[default.DISKTYPE] = boot_disk_type
can_ip_fwd = prop.setdefault(CAN_IP_FWD, DEFAULT_IP_FWD)
disks = prop.setdefault(default.DISKS, list())
local_ssd = prop.setdefault(default.LOCAL_SSD, 0)
if disks:
if create_disks_separately:
# Legacy alternative from when multiple disks on creation were not allowed
new_disks = prop.setdefault(default.DISK_RESOURCES, list())
SetDiskProperties(context, disks)
disks, prop[DISK_RESOURCES] = GenerateDisks(context, disks, new_disks)
else:
# All new disks (except local ssd) must provide a sourceImage or existing
# source. Add blank source image if non provided.
SetDiskProperties(context, disks, add_blank_src_img=True)
machine_type = prop.setdefault(MACHINETYPE, DEFAULT_MACHINETYPE)
metadata = prop.setdefault(METADATA, dict())
SetMetadataDefaults(metadata)
network = prop.setdefault(NETWORK, DEFAULT_NETWORK)
vm_name = MakeVMName(context)
provide_boot = prop.setdefault(PROVIDE_BOOT, DEFAULT_PROVIDE_BOOT)
tags = prop.setdefault(TAGS, dict([('items', [])]))
zone = prop.setdefault(ZONE, DEFAULT_ZONE)
has_external_ip = prop.get(HAS_EXTERNAL_IP, DEFAULT_HAS_EXTERNAL_IP)
static_ip = prop.get(STATIC_IP, DEFAULT_STATIC_IP)
nat_ip = prop.get(NAT_IP, None)
if provide_boot:
dev_mode = DEVIMAGE in prop and prop[DEVIMAGE]
src_image = common.MakeC2DImageLink(prop[SRCIMAGE], dev_mode)
boot_name = common.AutoName(context.env['name'], default.DISK, 'boot')
disk_size = prop.get(BOOTDISKSIZE, DEFAULT_BOOTDISKSIZE)
disk_type = common.MakeLocalComputeLink(context, DISKTYPE)
autodelete = prop.get(AUTODELETE_BOOTDISK, DEFAULT_AUTODELETE_BOOTDISK)
disks = PrependBootDisk(disks, boot_name, disk_type, disk_size, src_image,
autodelete)
if local_ssd:
disks = AppendLocalSSDDisks(context, disks, local_ssd)
machine_type = common.MakeLocalComputeLink(context, default.MACHINETYPE)
network = common.MakeGlobalComputeLink(context, default.NETWORK)
subnetwork = ''
if default.SUBNETWORK in prop:
subnetwork = common.MakeSubnetworkComputeLink(context, default.SUBNETWORK)
# To be consistent with Dev console and gcloud, service accounts need to be
# explicitly disabled
remove_scopes = prop[NO_SCOPE] if NO_SCOPE in prop else False
if remove_scopes and SERVICE_ACCOUNTS in prop:
prop.pop(SERVICE_ACCOUNTS)
else: # Make sure there is a default service account
prop.setdefault(SERVICE_ACCOUNTS, copy.deepcopy(DEFAULT_SERVICE_ACCOUNT))
resource = []
access_configs = []
if has_external_ip:
access_config = {'name': default.EXTERNAL, 'type': default.ONE_NAT}
access_configs.append(access_config)
if static_ip and nat_ip:
raise common.Error(
'staticIP=True and natIP cannot be specified at the same time')
if static_ip:
address_resource, nat_ip = MakeStaticAddress(vm_name, zone)
resource.append(address_resource)
if nat_ip:
access_config['natIP'] = nat_ip
else:
if static_ip:
raise common.Error('staticIP cannot be True when hasExternalIP is False')
if nat_ip:
raise common.Error(
'natIP must not be specified when hasExternalIP is False')
network_interfaces = []
if subnetwork:
network_interfaces.insert(0, {
'network': network,
'subnetwork': subnetwork,
'accessConfigs': access_configs
})
else:
network_interfaces.insert(0, {
'network': network,
'accessConfigs': access_configs
})
resource.insert(0, {
'name': vm_name,
'type': default.INSTANCE,
'properties': {
'zone': zone,
'machineType': machine_type,
'canIpForward': can_ip_fwd,
'disks': disks,
'networkInterfaces': network_interfaces,
'tags': tags,
'metadata': metadata,
}
})
# Pass through any additional properties to the VM
if SERVICE_ACCOUNTS in prop:
resource[0]['properties'].update({SERVICE_ACCOUNTS: prop[SERVICE_ACCOUNTS]})
if GUEST_ACCELERATORS in prop:
for accelerators in prop[GUEST_ACCELERATORS]:
accelerators['acceleratorType'] = common.MakeAcceleratorTypeLink(
context, accelerators['acceleratorType'])
resource[0]['properties'].update(
{GUEST_ACCELERATORS: prop[GUEST_ACCELERATORS]})
# GPUs cannot be attached to live migratable instances. See:
# https://cloud.google.com/compute/docs/gpus/#restrictions
resource[0]['properties'].update(
{'scheduling': {'onHostMaintenance': 'terminate'}})
return resource
def GenerateComputeVM(context):
"""Generates one VM instance resource."""
prop = context.properties
boot_disk_type = prop.setdefault(BOOTDISKTYPE, DEFAULT_DISKTYPE)
prop[default.DISKTYPE] = boot_disk_type
can_ip_fwd = prop.setdefault(CAN_IP_FWD, DEFAULT_IP_FWD)
disks = prop.setdefault(default.DISKS, list())
local_ssd = prop.setdefault(default.LOCAL_SSD, 0)
# Temporary alternative while multiple disks on creation is not allowed
if disks:
new_disks = prop.setdefault(default.DISK_RESOURCES, list())
disks, prop[DISK_RESOURCES] = GenerateDisks(context, disks, new_disks)
machine_type = prop.setdefault(MACHINETYPE, DEFAULT_MACHINETYPE)
metadata = prop.setdefault(METADATA, dict())
network = prop.setdefault(NETWORK, DEFAULT_NETWORK)
vm_name = MakeVMName(context)
provide_boot = prop.setdefault(PROVIDE_BOOT, DEFAULT_PROVIDE_BOOT)
tags = prop.setdefault(TAGS, dict([('items', [])]))
zone = prop.setdefault(ZONE, DEFAULT_ZONE)
has_external_ip = prop.get(HAS_EXTERNAL_IP, DEFAULT_HAS_EXTERNAL_IP)
static_ip = prop.get(STATIC_IP, DEFAULT_STATIC_IP)
nat_ip = prop.get(NAT_IP, None)
if provide_boot:
dev_mode = DEVIMAGE in prop and prop[DEVIMAGE]
src_image = common.MakeC2DImageLink(prop[SRCIMAGE], dev_mode)
boot_name = common.AutoName(context.env['name'], default.DISK, 'boot')
disk_size = prop.get(BOOTDISKSIZE, DEFAULT_BOOTDISKSIZE)
disk_type = common.MakeLocalComputeLink(context, DISKTYPE)
autodelete = prop.get(AUTODELETE_BOOTDISK, DEFAULT_AUTODELETE_BOOTDISK)
disks = PrependBootDisk(disks, boot_name, disk_type, disk_size,
src_image, autodelete)
if local_ssd:
disks = AppendLocalSSDDisks(context, disks, local_ssd)
machine_type = common.MakeLocalComputeLink(context, default.MACHINETYPE)
network = common.MakeGlobalComputeLink(context, default.NETWORK)
subnetwork = ''
if default.SUBNETWORK in prop:
subnetwork = common.MakeSubnetworkComputeLink(context,
default.SUBNETWORK)
# To be consistent with Dev console and gcloud, service accounts need to be
# explicitly disabled
remove_scopes = prop[NO_SCOPE] if NO_SCOPE in prop else False
if remove_scopes and SERVICE_ACCOUNTS in prop:
prop.pop(SERVICE_ACCOUNTS)
else: # Make sure there is a default service account
prop.setdefault(SERVICE_ACCOUNTS,
copy.deepcopy(DEFAULT_SERVICE_ACCOUNT))
resource = []
access_configs = []
if has_external_ip:
access_config = {'name': default.EXTERNAL, 'type': default.ONE_NAT}
access_configs.append(access_config)
if static_ip and nat_ip:
raise common.Error(
'staticIP=True and natIP cannot be specified at the same time')
if static_ip:
address_resource, nat_ip = MakeStaticAddress(vm_name, zone)
resource.append(address_resource)
if nat_ip:
access_config['natIP'] = nat_ip
else:
if static_ip:
raise common.Error(
'staticIP cannot be True when hasExternalIP is False')
if nat_ip:
raise common.Error(
'natIP must not be specified when hasExternalIP is False')
network_interfaces = []
if subnetwork:
network_interfaces.insert(
0, {
'network': network,
'subnetwork': subnetwork,
'accessConfigs': access_configs
})
else:
network_interfaces.insert(0, {
'network': network,
'accessConfigs': access_configs
})
resource.insert(
0, {
'name': vm_name,
'type': default.INSTANCE,
'properties': {
'zone': zone,
'machineType': machine_type,
'canIpForward': can_ip_fwd,
'disks': disks,
'networkInterfaces': network_interfaces,
'tags': tags,
'metadata': metadata,
}
})
# Pass through any additional property to the VM
if SERVICE_ACCOUNTS in prop:
resource[0]['properties'].update(
{SERVICE_ACCOUNTS: prop[SERVICE_ACCOUNTS]})
return resource
def GenerateComputeVM(context, create_disks_separately=True):
"""Generates one VM instance resource.
Args:
context: Template context dictionary.
create_disks_separately: When true (default), all new disk resources are
created as separate resources. This is legacy behaviour from when multiple
disks creation was not allowed in the disks property.
Returns:
dictionary representing instance resource.
"""
prop = context.properties
boot_disk_type = prop.setdefault(BOOTDISKTYPE, DEFAULT_DISKTYPE)
prop[default.DISKTYPE] = boot_disk_type
can_ip_fwd = prop.setdefault(CAN_IP_FWD, DEFAULT_IP_FWD)
disks = prop.setdefault(default.DISKS, list())
local_ssd = prop.setdefault(default.LOCAL_SSD, 0)
if disks:
if create_disks_separately:
# Legacy alternative from when multiple disks on creation were not allowed
new_disks = prop.setdefault(default.DISK_RESOURCES, list())
SetDiskProperties(context, disks)
disks, prop[DISK_RESOURCES] = GenerateDisks(
context, disks, new_disks)
else:
# All new disks (except local ssd) must provide a sourceImage or existing
# source. Add blank source image if non provided.
SetDiskProperties(context, disks, add_blank_src_img=True)
machine_type = prop.setdefault(MACHINETYPE, DEFAULT_MACHINETYPE)
metadata = prop.setdefault(METADATA, dict())
SetMetadataDefaults(metadata)
vm_name = MakeVMName(context)
provide_boot = prop.setdefault(PROVIDE_BOOT, DEFAULT_PROVIDE_BOOT)
tags = prop.setdefault(TAGS, dict([("items", [])]))
zone = prop.setdefault(ZONE, DEFAULT_ZONE)
if provide_boot:
dev_mode = DEVIMAGE in prop and prop[DEVIMAGE]
src_image = common.MakeC2DImageLink(prop[SRCIMAGE], dev_mode)
boot_name = common.AutoName(context.env["name"], default.DISK, "boot")
disk_size = prop.get(BOOTDISKSIZE, DEFAULT_BOOTDISKSIZE)
disk_type = common.MakeLocalComputeLink(context, DISKTYPE)
autodelete = prop.get(AUTODELETE_BOOTDISK, DEFAULT_AUTODELETE_BOOTDISK)
disks = PrependBootDisk(disks, boot_name, disk_type, disk_size,
src_image, autodelete)
if local_ssd:
disks = AppendLocalSSDDisks(context, disks, local_ssd)
machine_type = common.MakeLocalComputeLink(context, default.MACHINETYPE)
# To be consistent with Dev console and gcloud, service accounts need to be
# explicitly disabled
remove_scopes = prop[NO_SCOPE] if NO_SCOPE in prop else False
if remove_scopes and SERVICE_ACCOUNTS in prop:
prop.pop(SERVICE_ACCOUNTS)
else: # Make sure there is a default service account
prop.setdefault(SERVICE_ACCOUNTS,
copy.deepcopy(DEFAULT_SERVICE_ACCOUNT))
resource = []
resource.insert(
0,
{
"name": vm_name,
"type": default.INSTANCE,
"properties": {
"zone": zone,
"machineType": machine_type,
"canIpForward": can_ip_fwd,
"disks": disks,
"networkInterfaces": GetNetworkInterfaces(context),
"tags": tags,
"metadata": metadata,
},
},
)
# Pass through any additional properties to the VM
if SERVICE_ACCOUNTS in prop:
resource[0]["properties"].update(
{SERVICE_ACCOUNTS: prop[SERVICE_ACCOUNTS]})
if GUEST_ACCELERATORS in prop:
for accelerators in prop[GUEST_ACCELERATORS]:
accelerators["acceleratorType"] = common.MakeAcceleratorTypeLink(
context, accelerators["acceleratorType"])
resource[0]["properties"].update(
{GUEST_ACCELERATORS: prop[GUEST_ACCELERATORS]})
# GPUs cannot be attached to live migratable instances. See:
# https://cloud.google.com/compute/docs/gpus/#restrictions
resource[0]["properties"].update(
{"scheduling": {
"onHostMaintenance": "terminate"
}})
return resource
def GenerateComputeVM(context):
"""Generates one VM instance resource."""
name = context.env['name']
prop = context.properties
if SRCIMAGE not in prop:
raise common.Error('"%s" is a mandatory property' % SRCIMAGE)
boot_disk_type = prop.setdefault(BOOTDISKTYPE, DEFAULT_DISKTYPE)
prop[default.DISKTYPE] = boot_disk_type
can_ip_fwd = prop.setdefault(CAN_IP_FWD, DEFAULT_IP_FWD)
disks = prop.setdefault(default.DISKS, list())
# Temporary alternative while multiple disks on creation is not allowed
if disks:
new_disks = prop.setdefault(default.DISK_RESOURCES, list())
disks, prop[DISK_RESOURCES] = GenerateDisks(context, disks, new_disks)
machine_type = prop.setdefault(MACHINETYPE, DEFAULT_MACHINETYPE)
metadata = prop.setdefault(METADATA, dict())
network = prop.setdefault(NETWORK, DEFAULT_NETWORK)
named = INSTANCE_NAME in prop
provide_boot = prop.setdefault(PROVIDE_BOOT, DEFAULT_PROVIDE_BOOT)
tags = prop.setdefault(TAGS, dict([('items', [])]))
vm_name = prop[INSTANCE_NAME] if named else common.AutoName(
name, default.INSTANCE)
zone = prop.setdefault(ZONE, DEFAULT_ZONE)
if provide_boot:
dev_mode = DEVIMAGE in prop and prop[DEVIMAGE]
src_image = common.MakeC2DImageLink(prop[SRCIMAGE], dev_mode)
boot_name = common.AutoName(context.env['name'], default.DISK, 'boot')
disk_size = (prop[BOOTDISKSIZE]
if BOOTDISKSIZE in prop else DEFAULT_BOOTDISKSIZE)
disk_type = common.MakeLocalComputeLink(context, default.DISKTYPE)
disks = PrependBootDisk(disks, boot_name, disk_type, disk_size,
src_image)
machine_type = common.MakeLocalComputeLink(context, default.MACHINETYPE)
network = common.MakeGlobalComputeLink(context, default.NETWORK)
# To be consistent with Dev console and gcloud, service accounts need to be
# explicitly disabled
remove_scopes = prop[NO_SCOPE] if NO_SCOPE in prop else False
if remove_scopes and SERVICE_ACCOUNTS in prop:
prop.pop(SERVICE_ACCOUNTS)
else: # Make sure there is a default service account
prop.setdefault(SERVICE_ACCOUNTS,
copy.deepcopy(DEFAULT_SERVICE_ACCOUNT))
# pyformat: disable
resource = [{
'name': vm_name,
'type': default.INSTANCE,
'properties': {
'zone':
zone,
'machineType':
machine_type,
'canIpForward':
can_ip_fwd,
'disks':
disks,
'networkInterfaces': [{
'network':
network,
'accessConfigs': [{
'name': default.EXTERNAL,
'type': default.ONE_NAT
}]
}],
'tags':
tags,
'metadata':
metadata,
}
}]
# pyformat: enable
# Pass through any additional property to the VM
if SERVICE_ACCOUNTS in prop:
resource[0]['properties'].update(
{SERVICE_ACCOUNTS: prop[SERVICE_ACCOUNTS]})
return resource
def GenerateComputeVM(context, create_disks_separately=True):
"""Generates one VM instance resource.
Args:
context: Template context dictionary.
create_disks_separately: When true (default), all new disk resources are
created as separate resources. This is legacy behaviour from when multiple
disks creation was not allowed in the disks property.
Returns:
dictionary representing instance resource.
"""
prop = context.properties
boot_disk_type = prop.setdefault(BOOTDISKTYPE, DEFAULT_DISKTYPE)
prop[default.DISKTYPE] = boot_disk_type
can_ip_fwd = prop.setdefault(CAN_IP_FWD, DEFAULT_IP_FWD)
disks = prop.setdefault(default.DISKS, list())
local_ssd = prop.setdefault(default.LOCAL_SSD, 0)
project = context.env[default.PROJECT]
deployApiToGke = context.properties['deployApiToGke']
datashare_install_bucket_name = project + '-install-bucket'
datashare_ingestion_bucket_name = project + '-cds-bucket'
k8s_cluster_name = 'datashare-cluster-resource'
# gce_service_account = context.properties['gceServiceAccount']
if disks:
if create_disks_separately:
# Legacy alternative from when multiple disks on creation were not allowed
new_disks = prop.setdefault(default.DISK_RESOURCES, list())
SetDiskProperties(context, disks)
disks, prop[DISK_RESOURCES] = GenerateDisks(context, disks, new_disks)
else:
# All new disks (except local ssd) must provide a sourceImage or existing
# source. Add blank source image if non provided.
SetDiskProperties(context, disks, add_blank_src_img=True)
machine_type = prop.setdefault(MACHINETYPE, DEFAULT_MACHINETYPE)
metadata = prop.setdefault(METADATA, dict())
SetMetadataDefaults(metadata)
vm_name = MakeVMName(context)
provide_boot = prop.setdefault(PROVIDE_BOOT, DEFAULT_PROVIDE_BOOT)
tags = prop.setdefault(TAGS, dict([('items', [])]))
zone = prop.setdefault(ZONE, DEFAULT_ZONE)
if provide_boot:
dev_mode = DEVIMAGE in prop and prop[DEVIMAGE]
src_image = common.MakeC2DImageLink(prop[SRCIMAGE], dev_mode)
boot_name = common.AutoName(context.env['name'], default.DISK, 'boot')
disk_size = prop.get(BOOTDISKSIZE, DEFAULT_BOOTDISKSIZE)
disk_type = common.MakeLocalComputeLink(context, DISKTYPE)
autodelete = prop.get(AUTODELETE_BOOTDISK, DEFAULT_AUTODELETE_BOOTDISK)
disks = PrependBootDisk(disks, boot_name, disk_type, disk_size, src_image,
autodelete)
if local_ssd:
disks = AppendLocalSSDDisks(context, disks, local_ssd)
machine_type = common.MakeLocalComputeLink(context, default.MACHINETYPE)
# To be consistent with Dev console and gcloud, service accounts need to be
# explicitly disabled
remove_scopes = prop[NO_SCOPE] if NO_SCOPE in prop else False
if remove_scopes and SERVICE_ACCOUNTS in prop:
prop.pop(SERVICE_ACCOUNTS)
else: # Make sure there is a default service account
prop.setdefault(SERVICE_ACCOUNTS, copy.deepcopy(DEFAULT_SERVICE_ACCOUNT))
resource = []
resource.insert(
0, {
'name': vm_name,
'type': default.INSTANCE,
'metadata': {
'dependsOn': [
datashare_ingestion_bucket_name
]
},
'properties': {
'zone': zone,
'machineType': machine_type,
'canIpForward': can_ip_fwd,
'disks': disks,
'networkInterfaces': GetNetworkInterfaces(context),
'tags': tags,
'metadata': metadata,
}
})
if deployApiToGke:
resource[0]['metadata']['dependsOn'].append(k8s_cluster_name)
if context.properties['useRuntimeConfigWaiter']:
configName = context.properties['waiterConfigName']
resource[0]['metadata']['dependsOn'].append(configName)
resource[0]['metadata']['dependsOn'].append(datashare_install_bucket_name)
# Pass through any additional properties to the VM
if SERVICE_ACCOUNTS in prop:
resource[0]['properties'].update({SERVICE_ACCOUNTS: prop[SERVICE_ACCOUNTS]})
if GUEST_ACCELERATORS in prop:
for accelerators in prop[GUEST_ACCELERATORS]:
accelerators['acceleratorType'] = common.MakeAcceleratorTypeLink(
context, accelerators['acceleratorType'])
resource[0]['properties'].update(
{GUEST_ACCELERATORS: prop[GUEST_ACCELERATORS]})
# GPUs cannot be attached to live migratable instances. See:
# https://cloud.google.com/compute/docs/gpus/#restrictions
resource[0]['properties'].update(
{'scheduling': {'onHostMaintenance': 'terminate'}})
return resource
