def get_by_url(ce: CEClient, image_url: str): image_url = ImageUrl(image_url) if image_url.is_family: image_data = ce.get_image_from_family( family_name=image_url.name, project_id=image_url.project_id) else: res = ce.list_images(image_name=image_url.name, project_id=image_url.project_id) image_data = res[0] if res else None if not image_data: return None return Image(image_data)
def get_by_name(ce: CEClient, machine_name: str): """Returns an instance by its stack name.""" res = ce.list_instances(machine_name) if not res: return None return Instance(ce, res[0])
def get_by_name(ce: CEClient, disk_name: str): """Returns a disk by its name.""" res = ce.list_disks(disk_name) if not res: return None return Disk(ce, res[0])
def get_by_name(ce: CEClient, image_name: str): """Returns an image by its name.""" res = ce.list_images(image_name) if not res: return None return Image(res[0])
def get_by_name(ce: CEClient, snapshot_name: str): """Returns a snapshot by its name.""" res = ce.list_snapshots(snapshot_name) if not res: return None return Snapshot(res[0])
def __init__(self, machine_name: str, project_id: str, zone: str): self._dm = DMClient(project_id, zone) self._ce = CEClient(project_id, zone) self._rtc = RtcClient(project_id, zone) self._machine_name = machine_name self._stack_name = 'spotty-instance-' + machine_name # resource names self._INSTANCE_RESOURCE_NAME = machine_name self._DOCKER_WAITER_RESOURCE_NAME = machine_name + '-docker-waiter' self._DOCKER_STATUS_CONFIG_RESOURCE_NAME = machine_name + '-docker-status'
def check_gpu_configuration(ce: CEClient, gpu_parameters: dict): if not gpu_parameters: return # check GPU type accelerator_types = ce.get_accelerator_types() gpu_type = gpu_parameters['type'] if gpu_type not in accelerator_types: if accelerator_types: error_msg = 'GPU type "%s" is not supported in the "%s" zone.\nAvailable GPU types are: %s.' \ % (gpu_type, ce.zone, ', '.join(accelerator_types.keys())) else: error_msg = 'The "%s" zone doesn\'t support any GPU accelerators.' % ce.zone raise ValueError(error_msg) # check the number of GPUs is not exceed the maximum max_cards_per_instance = accelerator_types[gpu_parameters['type']] if gpu_parameters['count'] > max_cards_per_instance: raise ValueError( 'Maximum allowed number of cards per instance for the "%s" type is %d.' % (gpu_parameters['type'], max_cards_per_instance))
def create_disks(ce: CEClient, volumes: List[AbstractInstanceVolume], output: AbstractOutputWriter, dry_run: bool = False): disks_to_create = [] # do some checks and prepare disk parameters for i, volume in enumerate(volumes): if isinstance(volume, DiskVolume): # check if the disk already exists disk = Disk.get_by_name(ce, volume.disk_name) if disk: # check if the volume is available if not disk.is_available(): raise ValueError( 'Disk "%s" is not available (status: %s).' % (volume.disk_name, disk.status)) # check size of the volume if volume.size and (volume.size != disk.size): raise ValueError( 'Specified size for the "%s" volume (%dGB) doesn\'t match the size of the ' 'existing disk (%dGB).' % (volume.name, volume.size, disk.size)) output.write('- disk "%s" will be attached' % disk.name) else: # check if the snapshot exists snapshot = Snapshot.get_by_name(ce, volume.disk_name) if snapshot: # disk will be restored from the snapshot # check size of the volume if volume.size and (volume.size < snapshot.size): raise ValueError( 'Specified size for the "%s" volume (%dGB) is less than size of the ' 'snapshot (%dGB).' % (volume.name, volume.size, snapshot.size)) output.write( '- disk "%s" will be restored from the snapshot' % volume.disk_name) disks_to_create.append( (volume.disk_name, volume.size, snapshot.self_link)) else: # empty volume will be created, check that the size is specified if not volume.size: raise ValueError('Size for the new disk is required.') if volume.size < 10: raise ValueError( 'Size of a disk cannot be less than 10GB.') disks_to_create.append( (volume.disk_name, volume.size, None)) # create disks for disk_name, disk_size, snapshot_link in disks_to_create: if not dry_run: ce.create_disk(disk_name, disk_size, snapshot_link) output.write('- disk "%s" was created' % disk_name)
def __init__(self, image_name: str, project_id: str, zone: str): self._dm = DMClient(project_id, zone) self._ce = CEClient(project_id, zone) self._image_name = image_name self._stack_name = 'spotty-image-%s' % image_name
def __init__(self, instance_config: InstanceConfig): super().__init__(instance_config) self._project_name = instance_config.project_config.project_name self._credentials = GcpCredentials() self._ce = CEClient(self._credentials.project_id, instance_config.zone)
class InstanceDeployment(AbstractInstanceDeployment): instance_config: InstanceConfig def __init__(self, instance_config: InstanceConfig): super().__init__(instance_config) self._project_name = instance_config.project_config.project_name self._credentials = GcpCredentials() self._ce = CEClient(self._credentials.project_id, instance_config.zone) @property def stack_manager(self) -> InstanceStackManager: return InstanceStackManager(self.instance_config.machine_name, self._credentials.project_id, self.instance_config.zone) @property def ssh_key_manager(self) -> SshKeyManager: return SshKeyManager(self._project_name, self.instance_config.zone) def get_instance(self) -> Instance: return Instance.get_by_name(self._ce, self.instance_config.machine_name) def deploy(self, container_commands: DockerCommands, bucket_name: str, data_transfer: DataTransfer, output: AbstractOutputWriter, dry_run: bool = False): # check machine type if not self._ce.get_machine_types(self.instance_config.machine_type): raise ValueError( '"%s" machine type is not available in the "%s" zone.' % (self.instance_config.machine_type, self.instance_config.zone)) # check GPU configuration check_gpu_configuration(self._ce, self.instance_config.gpu) # remove the stack it it exists to make all the disks available stack_manager = self.stack_manager stack_manager.delete_stack(output=output) # sync the project with the S3 bucket if bucket_name is not None: output.write('Syncing the project with the bucket...') data_transfer.upload_local_to_bucket(bucket_name, dry_run=dry_run) # create volumes if self.instance_config.volumes: # create disks output.write('\nCreating disks...') with output.prefix(' '): create_disks(self._ce, self.instance_config.volumes, output=output, dry_run=dry_run) output.write('') # prepare Deployment Manager template output.write('Preparing the deployment template...') with output.prefix(' '): # get an image image_link = get_image(self._ce, self.instance_config.image_uri, self.instance_config.image_name).self_link # get or create an SSH key public_key_value = self.ssh_key_manager.get_public_key_value() # prepare the deployment template sync_project_cmd = data_transfer.get_download_bucket_to_instance_command( bucket_name=bucket_name) template = prepare_instance_template( instance_config=self.instance_config, docker_commands=container_commands, image_link=image_link, bucket_name=bucket_name, sync_project_cmd=sync_project_cmd, public_key_value=public_key_value, service_account_email=self._credentials.service_account_email, output=output, ) output.write('') # print information about the volumes output.write('Volumes:\n%s\n' % render_volumes_info_table( self.instance_config.volume_mounts, self.instance_config.volumes)) # create stack if not dry_run: stack_manager.create_stack(template, output=output) def delete(self, output: AbstractOutputWriter): self.stack_manager.delete_stack(output)
def __init__(self, project_name: str, instance_config: InstanceConfig): self._project_name = project_name self._instance_config = instance_config self._credentials = GcpCredentials() self._ce = CEClient(self._credentials.project_id, instance_config.zone)