def Provision(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
# Sort networks into a guaranteed order of creation based on dict key.
# There is a finite limit on the number of threads that are created to
# provision networks. Until support is added to provision resources in an
# order based on dependencies, this key ordering can be used to avoid
# deadlock by placing dependent networks later and their dependencies
# earlier. As an example, AWS stores both per-region and per-zone objects
# in this dict, and each per-zone object depends on a corresponding
# per-region object, so the per-region objects are given keys that come
# first when sorted.
networks = [self.networks[key] for key in sorted(self.networks.iterkeys())]
vm_util.RunThreaded(lambda net: net.Create(), networks)
if self.container_cluster:
self.container_cluster.Create()
if self.vms:
vm_util.RunThreaded(self.PrepareVm, self.vms)
sshable_vms = [vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS]
sshable_vm_groups = {}
for group_name, group_vms in self.vm_groups.iteritems():
sshable_vm_groups[group_name] = [
vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS
]
vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups)
if self.spark_service:
self.spark_service.Create()
if self.dpb_service:
self.dpb_service.Create()
if self.managed_relational_db:
self.managed_relational_db.client_vm = self.vms[0]
self.managed_relational_db.Create()
if self.cloud_tpu:
self.cloud_tpu.Create()
def Prepare(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
vm_util.RunThreaded(lambda net: net.Create(), self.networks.values())
if self.vms:
vm_util.RunThreaded(self.PrepareVm, self.vms)
if FLAGS.os_type != WINDOWS:
vm_util.GenerateSSHConfig(self.vms)
def Provision(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
# Sort networks into a guaranteed order of creation based on dict key.
# There is a finite limit on the number of threads that are created to
# provision networks. Until support is added to provision resources in an
# order based on dependencies, this key ordering can be used to avoid
# deadlock by placing dependent networks later and their dependencies
# earlier. As an example, AWS stores both per-region and per-zone objects
# in this dict, and each per-zone object depends on a corresponding
# per-region object, so the per-region objects are given keys that come
# first when sorted.
networks = [self.networks[key] for key in sorted(self.networks.iterkeys())]
vm_util.RunThreaded(lambda net: net.Create(), networks)
if self.container_registry:
self.container_registry.Create()
for container_spec in self.container_specs.itervalues():
if container_spec.static_image:
continue
container_spec.image = self.container_registry.GetOrBuild(
container_spec.image)
if self.container_cluster:
self.container_cluster.Create()
# do after network setup but before VM created
if self.nfs_service:
self.nfs_service.Create()
if self.vms:
vm_util.RunThreaded(self.PrepareVm, self.vms)
sshable_vms = [vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS]
sshable_vm_groups = {}
for group_name, group_vms in self.vm_groups.iteritems():
sshable_vm_groups[group_name] = [
vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS
]
vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups)
if self.spark_service:
self.spark_service.Create()
if self.dpb_service:
self.dpb_service.Create()
if self.managed_relational_db:
self.managed_relational_db.client_vm = self.vms[0]
self.managed_relational_db.Create()
if self.cloud_tpu:
self.cloud_tpu.Create()
if self.edw_service:
if not self.edw_service.user_managed:
# The benchmark creates the Redshift cluster's subnet group in the
# already provisioned virtual private cloud (vpc).
for network in networks:
if network.__class__.__name__ == 'AwsNetwork':
self.config.edw_service.subnet_id = network.subnet.id
self.edw_service.Create()
if self.cloud_redis:
self.config.cloud_redis.client_vm = self.vms[0]
self.cloud_redis.Create()
def Prepare(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
if self.networks:
prepare_args = [self.networks[zone] for zone in self.networks]
vm_util.RunThreaded(self.PrepareNetwork, prepare_args)
if self.vms:
prepare_args = [((vm, self.firewall), {}) for vm in self.vms]
vm_util.RunThreaded(self.PrepareVm, prepare_args)
vm_util.GenerateSSHConfig(self.vms)
def Prepare(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
prepare_args = network.BaseNetwork.networks.values()
vm_util.RunThreaded(self.PrepareNetwork, prepare_args)
if self.vms:
prepare_args = [((vm, self.firewall), {}) for vm in self.vms]
vm_util.RunThreaded(self.PrepareVm, prepare_args)
if FLAGS.os_type != WINDOWS:
vm_util.GenerateSSHConfig(self.vms)
def Provision(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
# Sort networks into a guaranteed order of creation based on dict key.
# There is a finite limit on the number of threads that are created to
# provision networks. Until support is added to provision resources in an
# order based on dependencies, this key ordering can be used to avoid
# deadlock by placing dependent networks later and their dependencies
# earlier. As an example, AWS stores both per-region and per-zone objects
# in this dict, and each per-zone object depends on a corresponding
# per-region object, so the per-region objects are given keys that come
# first when sorted.
networks = [self.networks[key] for key in sorted(self.networks.iterkeys())]
vm_util.RunThreaded(lambda net: net.Create(), networks)
if self.vms:
vm_util.RunThreaded(self.PrepareVm, self.vms)
if FLAGS.os_type != os_types.WINDOWS:
vm_util.GenerateSSHConfig(self)
def Provision(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
# Create capacity reservations if the cloud supports it. Note that the
# capacity reservation class may update the VMs themselves. This is true
# on AWS, because the VM needs to be aware of the capacity reservation id
# before its Create() method is called. Furthermore, if the user does not
# specify an AWS zone, but a region instead, the AwsCapacityReservation
# class will make a reservation in a zone that has sufficient capacity.
# In this case the VM's zone attribute, and the VMs network instance
# need to be updated as well.
if self.capacity_reservations:
vm_util.RunThreaded(lambda res: res.Create(), self.capacity_reservations)
# Sort networks into a guaranteed order of creation based on dict key.
# There is a finite limit on the number of threads that are created to
# provision networks. Until support is added to provision resources in an
# order based on dependencies, this key ordering can be used to avoid
# deadlock by placing dependent networks later and their dependencies
# earlier.
networks = [
self.networks[key] for key in sorted(six.iterkeys(self.networks))
]
vm_util.RunThreaded(lambda net: net.Create(), networks)
# VPC peering is currently only supported for connecting 2 VPC networks
if self.vpc_peering:
if len(networks) > 2:
raise errors.Error(
'Networks of size %d are not currently supported.' %
(len(networks)))
# Ignore Peering for one network
elif len(networks) == 2:
networks[0].Peer(networks[1])
if self.container_registry:
self.container_registry.Create()
for container_spec in six.itervalues(self.container_specs):
if container_spec.static_image:
continue
container_spec.image = self.container_registry.GetOrBuild(
container_spec.image)
if self.container_cluster:
self.container_cluster.Create()
# do after network setup but before VM created
if self.nfs_service and self.nfs_service.CLOUD != nfs_service.UNMANAGED:
self.nfs_service.Create()
if self.smb_service:
self.smb_service.Create()
for placement_group_object in self.placement_groups.values():
placement_group_object.Create()
if self.vms:
# We separate out creating, booting, and preparing the VMs into two phases
# so that we don't slow down the creation of all the VMs by running
# commands on the VMs that booted.
vm_util.RunThreaded(
self.CreateAndBootVm,
self.vms,
post_task_delay=FLAGS.create_and_boot_post_task_delay)
if self.nfs_service and self.nfs_service.CLOUD == nfs_service.UNMANAGED:
self.nfs_service.Create()
vm_util.RunThreaded(self.PrepareVmAfterBoot, self.vms)
sshable_vms = [
vm for vm in self.vms if vm.OS_TYPE not in os_types.WINDOWS_OS_TYPES
]
sshable_vm_groups = {}
for group_name, group_vms in six.iteritems(self.vm_groups):
sshable_vm_groups[group_name] = [
vm for vm in group_vms
if vm.OS_TYPE not in os_types.WINDOWS_OS_TYPES
]
vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups)
if self.spark_service:
self.spark_service.Create()
if self.dpb_service:
self.dpb_service.Create()
if hasattr(self, 'relational_db') and self.relational_db:
self.relational_db.SetVms(self.vm_groups)
self.relational_db.Create()
if self.spanner:
self.spanner.Create()
if self.tpus:
vm_util.RunThreaded(lambda tpu: tpu.Create(), self.tpus)
if self.edw_service:
if (not self.edw_service.user_managed and
self.edw_service.SERVICE_TYPE == 'redshift'):
# The benchmark creates the Redshift cluster's subnet group in the
# already provisioned virtual private cloud (vpc).
for network in networks:
if network.__class__.__name__ == 'AwsNetwork':
self.edw_service.cluster_subnet_group.subnet_id = network.subnet.id
self.edw_service.Create()
if self.vpn_service:
self.vpn_service.Create()
def Provision(self):
"""Prepares the VMs and networks necessary for the benchmark to run."""
# Create capacity reservations if the cloud supports it. Note that the
# capacity reservation class may update the VMs themselves. This is true
# on AWS, because the VM needs to be aware of the capacity resrevation id
# before its Create() method is called. Furthermore, if the user does not
# specify an AWS zone, but a region instead, the AwsCapacityReservation
# class will make a reservation in a zone that has sufficient capacity.
# In this case the VM's zone attribute, and the VMs network instance
# need to be updated as well.
if self.capacity_reservations:
vm_util.RunThreaded(lambda res: res.Create(),
self.capacity_reservations)
# Sort networks into a guaranteed order of creation based on dict key.
# There is a finite limit on the number of threads that are created to
# provision networks. Until support is added to provision resources in an
# order based on dependencies, this key ordering can be used to avoid
# deadlock by placing dependent networks later and their dependencies
# earlier. As an example, AWS stores both per-region and per-zone objects
# in this dict, and each per-zone object depends on a corresponding
# per-region object, so the per-region objects are given keys that come
# first when sorted.
networks = [
self.networks[key] for key in sorted(self.networks.iterkeys())
]
vm_util.RunThreaded(lambda net: net.Create(), networks)
if self.container_registry:
self.container_registry.Create()
for container_spec in self.container_specs.itervalues():
if container_spec.static_image:
continue
container_spec.image = self.container_registry.GetOrBuild(
container_spec.image)
if self.container_cluster:
self.container_cluster.Create()
# do after network setup but before VM created
if self.nfs_service:
self.nfs_service.Create()
if self.vms:
vm_util.RunThreaded(self.PrepareVm, self.vms)
sshable_vms = [
vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS
]
sshable_vm_groups = {}
for group_name, group_vms in self.vm_groups.iteritems():
sshable_vm_groups[group_name] = [
vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS
]
vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups)
if self.spark_service:
self.spark_service.Create()
if self.dpb_service:
self.dpb_service.Create()
if self.managed_relational_db:
self.managed_relational_db.client_vm = self.vms[0]
self.managed_relational_db.Create()
if self.tpus:
vm_util.RunThreaded(lambda tpu: tpu.Create(), self.tpus)
if self.edw_service:
if not self.edw_service.user_managed:
# The benchmark creates the Redshift cluster's subnet group in the
# already provisioned virtual private cloud (vpc).
for network in networks:
if network.__class__.__name__ == 'AwsNetwork':
self.config.edw_service.subnet_id = network.subnet.id
self.edw_service.Create()
if self.cloud_redis:
self.config.cloud_redis.client_vm = self.vms[0]
self.cloud_redis.Create()
