def Provision(self): """Prepares the VMs and networks necessary for the benchmark to run.""" # Sort networks into a guaranteed order of creation based on dict key. # There is a finite limit on the number of threads that are created to # provision networks. Until support is added to provision resources in an # order based on dependencies, this key ordering can be used to avoid # deadlock by placing dependent networks later and their dependencies # earlier. As an example, AWS stores both per-region and per-zone objects # in this dict, and each per-zone object depends on a corresponding # per-region object, so the per-region objects are given keys that come # first when sorted. networks = [self.networks[key] for key in sorted(self.networks.iterkeys())] vm_util.RunThreaded(lambda net: net.Create(), networks) if self.container_cluster: self.container_cluster.Create() if self.vms: vm_util.RunThreaded(self.PrepareVm, self.vms) sshable_vms = [vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS] sshable_vm_groups = {} for group_name, group_vms in self.vm_groups.iteritems(): sshable_vm_groups[group_name] = [ vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS ] vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups) if self.spark_service: self.spark_service.Create() if self.dpb_service: self.dpb_service.Create() if self.managed_relational_db: self.managed_relational_db.client_vm = self.vms[0] self.managed_relational_db.Create() if self.cloud_tpu: self.cloud_tpu.Create()
def Prepare(self): """Prepares the VMs and networks necessary for the benchmark to run.""" vm_util.RunThreaded(lambda net: net.Create(), self.networks.values()) if self.vms: vm_util.RunThreaded(self.PrepareVm, self.vms) if FLAGS.os_type != WINDOWS: vm_util.GenerateSSHConfig(self.vms)
def Provision(self): """Prepares the VMs and networks necessary for the benchmark to run.""" # Sort networks into a guaranteed order of creation based on dict key. # There is a finite limit on the number of threads that are created to # provision networks. Until support is added to provision resources in an # order based on dependencies, this key ordering can be used to avoid # deadlock by placing dependent networks later and their dependencies # earlier. As an example, AWS stores both per-region and per-zone objects # in this dict, and each per-zone object depends on a corresponding # per-region object, so the per-region objects are given keys that come # first when sorted. networks = [self.networks[key] for key in sorted(self.networks.iterkeys())] vm_util.RunThreaded(lambda net: net.Create(), networks) if self.container_registry: self.container_registry.Create() for container_spec in self.container_specs.itervalues(): if container_spec.static_image: continue container_spec.image = self.container_registry.GetOrBuild( container_spec.image) if self.container_cluster: self.container_cluster.Create() # do after network setup but before VM created if self.nfs_service: self.nfs_service.Create() if self.vms: vm_util.RunThreaded(self.PrepareVm, self.vms) sshable_vms = [vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS] sshable_vm_groups = {} for group_name, group_vms in self.vm_groups.iteritems(): sshable_vm_groups[group_name] = [ vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS ] vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups) if self.spark_service: self.spark_service.Create() if self.dpb_service: self.dpb_service.Create() if self.managed_relational_db: self.managed_relational_db.client_vm = self.vms[0] self.managed_relational_db.Create() if self.cloud_tpu: self.cloud_tpu.Create() if self.edw_service: if not self.edw_service.user_managed: # The benchmark creates the Redshift cluster's subnet group in the # already provisioned virtual private cloud (vpc). for network in networks: if network.__class__.__name__ == 'AwsNetwork': self.config.edw_service.subnet_id = network.subnet.id self.edw_service.Create() if self.cloud_redis: self.config.cloud_redis.client_vm = self.vms[0] self.cloud_redis.Create()
def Prepare(self): """Prepares the VMs and networks necessary for the benchmark to run.""" if self.networks: prepare_args = [self.networks[zone] for zone in self.networks] vm_util.RunThreaded(self.PrepareNetwork, prepare_args) if self.vms: prepare_args = [((vm, self.firewall), {}) for vm in self.vms] vm_util.RunThreaded(self.PrepareVm, prepare_args) vm_util.GenerateSSHConfig(self.vms)
def Prepare(self): """Prepares the VMs and networks necessary for the benchmark to run.""" prepare_args = network.BaseNetwork.networks.values() vm_util.RunThreaded(self.PrepareNetwork, prepare_args) if self.vms: prepare_args = [((vm, self.firewall), {}) for vm in self.vms] vm_util.RunThreaded(self.PrepareVm, prepare_args) if FLAGS.os_type != WINDOWS: vm_util.GenerateSSHConfig(self.vms)
def Provision(self): """Prepares the VMs and networks necessary for the benchmark to run.""" # Sort networks into a guaranteed order of creation based on dict key. # There is a finite limit on the number of threads that are created to # provision networks. Until support is added to provision resources in an # order based on dependencies, this key ordering can be used to avoid # deadlock by placing dependent networks later and their dependencies # earlier. As an example, AWS stores both per-region and per-zone objects # in this dict, and each per-zone object depends on a corresponding # per-region object, so the per-region objects are given keys that come # first when sorted. networks = [self.networks[key] for key in sorted(self.networks.iterkeys())] vm_util.RunThreaded(lambda net: net.Create(), networks) if self.vms: vm_util.RunThreaded(self.PrepareVm, self.vms) if FLAGS.os_type != os_types.WINDOWS: vm_util.GenerateSSHConfig(self)
def Provision(self): """Prepares the VMs and networks necessary for the benchmark to run.""" # Create capacity reservations if the cloud supports it. Note that the # capacity reservation class may update the VMs themselves. This is true # on AWS, because the VM needs to be aware of the capacity reservation id # before its Create() method is called. Furthermore, if the user does not # specify an AWS zone, but a region instead, the AwsCapacityReservation # class will make a reservation in a zone that has sufficient capacity. # In this case the VM's zone attribute, and the VMs network instance # need to be updated as well. if self.capacity_reservations: vm_util.RunThreaded(lambda res: res.Create(), self.capacity_reservations) # Sort networks into a guaranteed order of creation based on dict key. # There is a finite limit on the number of threads that are created to # provision networks. Until support is added to provision resources in an # order based on dependencies, this key ordering can be used to avoid # deadlock by placing dependent networks later and their dependencies # earlier. networks = [ self.networks[key] for key in sorted(six.iterkeys(self.networks)) ] vm_util.RunThreaded(lambda net: net.Create(), networks) # VPC peering is currently only supported for connecting 2 VPC networks if self.vpc_peering: if len(networks) > 2: raise errors.Error( 'Networks of size %d are not currently supported.' % (len(networks))) # Ignore Peering for one network elif len(networks) == 2: networks[0].Peer(networks[1]) if self.container_registry: self.container_registry.Create() for container_spec in six.itervalues(self.container_specs): if container_spec.static_image: continue container_spec.image = self.container_registry.GetOrBuild( container_spec.image) if self.container_cluster: self.container_cluster.Create() # do after network setup but before VM created if self.nfs_service and self.nfs_service.CLOUD != nfs_service.UNMANAGED: self.nfs_service.Create() if self.smb_service: self.smb_service.Create() for placement_group_object in self.placement_groups.values(): placement_group_object.Create() if self.vms: # We separate out creating, booting, and preparing the VMs into two phases # so that we don't slow down the creation of all the VMs by running # commands on the VMs that booted. vm_util.RunThreaded( self.CreateAndBootVm, self.vms, post_task_delay=FLAGS.create_and_boot_post_task_delay) if self.nfs_service and self.nfs_service.CLOUD == nfs_service.UNMANAGED: self.nfs_service.Create() vm_util.RunThreaded(self.PrepareVmAfterBoot, self.vms) sshable_vms = [ vm for vm in self.vms if vm.OS_TYPE not in os_types.WINDOWS_OS_TYPES ] sshable_vm_groups = {} for group_name, group_vms in six.iteritems(self.vm_groups): sshable_vm_groups[group_name] = [ vm for vm in group_vms if vm.OS_TYPE not in os_types.WINDOWS_OS_TYPES ] vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups) if self.spark_service: self.spark_service.Create() if self.dpb_service: self.dpb_service.Create() if hasattr(self, 'relational_db') and self.relational_db: self.relational_db.SetVms(self.vm_groups) self.relational_db.Create() if self.spanner: self.spanner.Create() if self.tpus: vm_util.RunThreaded(lambda tpu: tpu.Create(), self.tpus) if self.edw_service: if (not self.edw_service.user_managed and self.edw_service.SERVICE_TYPE == 'redshift'): # The benchmark creates the Redshift cluster's subnet group in the # already provisioned virtual private cloud (vpc). for network in networks: if network.__class__.__name__ == 'AwsNetwork': self.edw_service.cluster_subnet_group.subnet_id = network.subnet.id self.edw_service.Create() if self.vpn_service: self.vpn_service.Create()
def Provision(self): """Prepares the VMs and networks necessary for the benchmark to run.""" # Create capacity reservations if the cloud supports it. Note that the # capacity reservation class may update the VMs themselves. This is true # on AWS, because the VM needs to be aware of the capacity resrevation id # before its Create() method is called. Furthermore, if the user does not # specify an AWS zone, but a region instead, the AwsCapacityReservation # class will make a reservation in a zone that has sufficient capacity. # In this case the VM's zone attribute, and the VMs network instance # need to be updated as well. if self.capacity_reservations: vm_util.RunThreaded(lambda res: res.Create(), self.capacity_reservations) # Sort networks into a guaranteed order of creation based on dict key. # There is a finite limit on the number of threads that are created to # provision networks. Until support is added to provision resources in an # order based on dependencies, this key ordering can be used to avoid # deadlock by placing dependent networks later and their dependencies # earlier. As an example, AWS stores both per-region and per-zone objects # in this dict, and each per-zone object depends on a corresponding # per-region object, so the per-region objects are given keys that come # first when sorted. networks = [ self.networks[key] for key in sorted(self.networks.iterkeys()) ] vm_util.RunThreaded(lambda net: net.Create(), networks) if self.container_registry: self.container_registry.Create() for container_spec in self.container_specs.itervalues(): if container_spec.static_image: continue container_spec.image = self.container_registry.GetOrBuild( container_spec.image) if self.container_cluster: self.container_cluster.Create() # do after network setup but before VM created if self.nfs_service: self.nfs_service.Create() if self.vms: vm_util.RunThreaded(self.PrepareVm, self.vms) sshable_vms = [ vm for vm in self.vms if vm.OS_TYPE != os_types.WINDOWS ] sshable_vm_groups = {} for group_name, group_vms in self.vm_groups.iteritems(): sshable_vm_groups[group_name] = [ vm for vm in group_vms if vm.OS_TYPE != os_types.WINDOWS ] vm_util.GenerateSSHConfig(sshable_vms, sshable_vm_groups) if self.spark_service: self.spark_service.Create() if self.dpb_service: self.dpb_service.Create() if self.managed_relational_db: self.managed_relational_db.client_vm = self.vms[0] self.managed_relational_db.Create() if self.tpus: vm_util.RunThreaded(lambda tpu: tpu.Create(), self.tpus) if self.edw_service: if not self.edw_service.user_managed: # The benchmark creates the Redshift cluster's subnet group in the # already provisioned virtual private cloud (vpc). for network in networks: if network.__class__.__name__ == 'AwsNetwork': self.config.edw_service.subnet_id = network.subnet.id self.edw_service.Create() if self.cloud_redis: self.config.cloud_redis.client_vm = self.vms[0] self.cloud_redis.Create()