def _create_container_group(self, resource_group_name, name, location, image, memory, cpu, algorithm, asset, input_mount_point, output_moint_point, account_name, account_key, share_name_input, share_name_output): # setup default values result_file = 'result-' + str(int(time.time())) command = ['python', input_mount_point + '/' + algorithm, input_mount_point + '/' + asset, output_moint_point + '/' + result_file] environment_variables = None az_file_input = AzureFileVolume(share_name=share_name_input, storage_account_name=account_name, storage_account_key=account_key, ) az_file_output = AzureFileVolume(share_name=share_name_output, storage_account_name=account_name, storage_account_key=account_key, ) volume = [Volume(name=share_name_input, azure_file=az_file_input), Volume(name=share_name_output, azure_file=az_file_output)] volume_mount = [VolumeMount(name=share_name_input, mount_path=input_mount_point), VolumeMount(name=share_name_output, mount_path=output_moint_point)] # set memory and cpu container_resource_requests = ResourceRequests(memory_in_gb=memory, cpu=cpu) container_resource_requirements = ResourceRequirements(requests=container_resource_requests) container = Container(name=name, image=image, resources=container_resource_requirements, command=command, environment_variables=environment_variables, volume_mounts=volume_mount, ) # defaults for container group cgroup_os_type = OperatingSystemTypes.linux cgroup = ContainerGroup(location=location, containers=[container], os_type=cgroup_os_type, restart_policy=ContainerGroupRestartPolicy.never, volumes=volume, ) self.client.container_groups.create_or_update(resource_group_name, name, cgroup) return result_file
def create_container_group(az_conf, client, resource_group_name, name, location, image, memory, cpu): # Start new containers # setup default values port = 80 container_resource_requirements = None command = None environment_variables = None volume_mount = [VolumeMount(name='config', mount_path='/mnt/conf'),VolumeMount(name='data', mount_path='/mnt/data')] # data_volume_mount = [VolumeMount(name='data', mount_path='/mnt/data')] # set memory and cpu container_resource_requests = ResourceRequests(memory_in_gb = memory, cpu = cpu) container_resource_requirements = ResourceRequirements(requests = container_resource_requests) az_config_file_volume = AzureFileVolume(share_name='config', storage_account_name=az_conf['storage_account_name'], storage_account_key=az_conf['storage_account_key']) az_data_file_volume = AzureFileVolume(share_name='data', storage_account_name=az_conf['storage_account_name'], storage_account_key=az_conf['storage_account_key']) #volume_mount = VolumeMount(name='config', mount_path='/mnt/config') volumes = [Volume(name='config', azure_file=az_config_file_volume),Volume(name='data', azure_file=az_data_file_volume)] container = Container(name = name, image = image, resources = container_resource_requirements, command = command, ports = [ContainerPort(port=port)], environment_variables = environment_variables, volume_mounts = volume_mount) # defaults for container group cgroup_os_type = OperatingSystemTypes.linux cgroup_restart_policy = ContainerGroupRestartPolicy.never image_registry_credential = ImageRegistryCredential(server=az_conf['container_registry'], username=az_conf['container_registry_user'], password=az_conf['container_registry_pwd']) cgroup = ContainerGroup(location = location, containers = [container], os_type = cgroup_os_type, image_registry_credentials = [image_registry_credential], restart_policy = cgroup_restart_policy, volumes = volumes) toto = client.container_groups.create_or_update(resource_group_name, name, cgroup) #print(toto.containers) """
def _create_gitrepo_volume_mount(gitrepo_volume, gitrepo_mount_path): """Create Git Repo volume mount. """ if gitrepo_mount_path: if not gitrepo_volume: raise CLIError('Please specify --gitrepo-url (--gitrepo-dir --gitrepo-revision) ' 'to enable Git Repo volume mount.') return VolumeMount(name=GITREPO_VOLUME_NAME, mount_path=gitrepo_mount_path)
def _create_secrets_volume_mount(secrets_volume, secrets_mount_path): """Create secrets volume mount. """ if secrets_volume: if not secrets_mount_path: raise CLIError('Please specify --secrets --secrets-mount-path ' 'to enable secrets volume mount.') return VolumeMount(name=SECRETS_VOLUME_NAME, mount_path=secrets_mount_path)
def _create_azure_file_volume_mount(azure_file_volume, azure_file_volume_mount_path): """Create Azure File volume mount. """ if azure_file_volume_mount_path: if not azure_file_volume: raise CLIError('Please specify --azure-file-volume-share-name --azure-file-volume-account-name --azure-file-volume-account-key ' 'to enable Azure File volume mount.') return VolumeMount(name=AZURE_FILE_VOLUME_NAME, mount_path=azure_file_volume_mount_path)
def create_container(name, image, pr_number, user_id, resource_group, port=None, location='westus'): from azure.mgmt.containerinstance.models import (ContainerGroup, Container, ContainerPort, Port, IpAddress, ImageRegistryCredential, ResourceRequirements, ResourceRequests, ContainerGroupNetworkProtocol, OperatingSystemTypes, EnvironmentVariable, Volume, AzureFileVolume, VolumeMount) from random import randint import secrets pr_number = str(pr_number) if port is None: port = randint(1000, 65000) instance_token = secrets.token_urlsafe(256) environment_variables = [EnvironmentVariable('PR_NUM', pr_number), EnvironmentVariable('PORT', port), EnvironmentVariable('INSTANCE_TOKEN', instance_token)] tags = {'userId': user_id, 'prNumber': pr_number} container_resource_requests = ResourceRequests(memory_in_gb='1.5', cpu='1') container_resource_requirements = ResourceRequirements(requests=container_resource_requests) container = Container(name=name, image=image, resources=container_resource_requirements, ports=[ContainerPort(port=port)], environment_variables=environment_variables, volume_mounts=[VolumeMount('volume1', '/cert', read_only=True)]) cgroup_ip_address = IpAddress(ports=[Port(protocol=ContainerGroupNetworkProtocol.tcp, port=port)]) cgroup_os_type = OperatingSystemTypes.linux afv = AzureFileVolume(SHARE_NAME, SHARE_STORAGE_ACCOUNT_NAME, read_only=True, storage_account_key=SHARE_STORAGE_ACCOUNT_KEY) cgroup = ContainerGroup(location=location, containers=[container], os_type=cgroup_os_type, ip_address=cgroup_ip_address, tags=tags, volumes=[Volume('volume1', afv)]) return get_aci_client().container_groups.create_or_update(resource_group, name, cgroup)
def _create_container_group(client, resource_group_name, name, location, image, memory, cpu, volumes): # Start new containers # setup default values port = 80 container_resource_requirements = None command = None environment_variables = None if volumes: az_volume_mount = [] az_volumes = [] for volume in volumes: az_volume_mount = az_volume_mount + [ VolumeMount(name=str(volume['name']), mount_path=volume['mount_path']) ] az_file = AzureFileVolume( share_name=str(volume['name']), storage_account_name=Variable.get("STORAGE_ACCOUNT_NAME"), storage_account_key=Variable.get("STORAGE_ACCOUNT_KEY_PASSWD")) az_volumes = az_volumes + [ Volume(name=str(volume['name']), azure_file=az_file) ] # set memory and cpu container_resource_requests = ResourceRequests(memory_in_gb=memory, cpu=cpu) container_resource_requirements = ResourceRequirements( requests=container_resource_requests) container = Container(name=name, image=image, resources=container_resource_requirements, command=command, ports=[ContainerPort(port=port)], environment_variables=environment_variables, volume_mounts=az_volume_mount) # defaults for container group cgroup_os_type = OperatingSystemTypes.linux cgroup_restart_policy = ContainerGroupRestartPolicy.never image_registry_credential = ImageRegistryCredential( server=Variable.get("CONTAINER_REGISTRY"), username=Variable.get("CONTAINER_REGISTRY_USER"), password=Variable.get("CONTAINER_REGISTRY_PASSWD")) cgroup = ContainerGroup( location=location, containers=[container], os_type=cgroup_os_type, image_registry_credentials=[image_registry_credential], restart_policy=cgroup_restart_policy, volumes=az_volumes) results = client.container_groups.create_or_update(resource_group_name, name, cgroup) return (results)
def prepare_azure_volumes(self, afs_volumes: list = []): volumes = [] volume_mounts = [] for afs in afs_volumes: assert afs['afs_name'] is not None assert afs['afs_key'] is not None assert afs['afs_share'] is not None assert afs['mount_path'] is not None assert afs['name'] is not None az_volume = AzureFileVolume(share_name=afs['afs_share'], storage_account_name=afs['afs_name'], storage_account_key=afs['afs_key']) volumes.append(Volume(name=afs['name'], azure_file=az_volume)) volume_mounts.append( VolumeMount(name=afs['name'], mount_path=afs['mount_path'])) return volumes, volume_mounts
def execute(self, context: dict) -> int: # Check name again in case it was templated. self._check_name(self.name) self._ci_hook = AzureContainerInstanceHook(self.ci_conn_id) if self.fail_if_exists: self.log.info("Testing if container group already exists") if self._ci_hook.exists(self.resource_group, self.name): raise AirflowException("Container group exists") if self.registry_conn_id: registry_hook = AzureContainerRegistryHook(self.registry_conn_id) image_registry_credentials: Optional[list] = [ registry_hook.connection, ] else: image_registry_credentials = None environment_variables = [] for key, value in self.environment_variables.items(): if key in self.secured_variables: e = EnvironmentVariable(name=key, secure_value=value) else: e = EnvironmentVariable(name=key, value=value) environment_variables.append(e) volumes: List[Union[Volume, Volume]] = [] volume_mounts: List[Union[VolumeMount, VolumeMount]] = [] for conn_id, account_name, share_name, mount_path, read_only in self.volumes: hook = AzureContainerVolumeHook(conn_id) mount_name = "mount-%d" % len(volumes) volumes.append( hook.get_file_volume(mount_name, share_name, account_name, read_only)) volume_mounts.append( VolumeMount(name=mount_name, mount_path=mount_path, read_only=read_only)) exit_code = 1 try: self.log.info("Starting container group with %.1f cpu %.1f mem", self.cpu, self.memory_in_gb) if self.gpu: self.log.info("GPU count: %.1f, GPU SKU: %s", self.gpu.count, self.gpu.sku) resources = ResourceRequirements(requests=ResourceRequests( memory_in_gb=self.memory_in_gb, cpu=self.cpu, gpu=self.gpu)) if self.ip_address and not self.ports: self.ports = [ContainerPort(port=80)] self.log.info( "Default port set. Container will listen on port 80") container = Container( name=self.name, image=self.image, resources=resources, command=self.command, environment_variables=environment_variables, volume_mounts=volume_mounts, ports=self.ports, ) container_group = ContainerGroup( location=self.region, containers=[ container, ], image_registry_credentials=image_registry_credentials, volumes=volumes, restart_policy=self.restart_policy, os_type=self.os_type, tags=self.tags, ip_address=self.ip_address, ) self._ci_hook.create_or_update(self.resource_group, self.name, container_group) self.log.info("Container group started %s/%s", self.resource_group, self.name) exit_code = self._monitor_logging(self.resource_group, self.name) self.log.info("Container had exit code: %s", exit_code) if exit_code != 0: raise AirflowException( f"Container had a non-zero exit code, {exit_code}") return exit_code except CloudError: self.log.exception("Could not start container group") raise AirflowException("Could not start container group") finally: if exit_code == 0 or self.remove_on_error: self.on_kill()
def test_container_instance(self, resource_group, location): container_group_name = self.get_resource_name('pycontainer') image = 'alpine:latest' os_type = 'Linux' cpu = 1 memory = 1 restart_policy = 'OnFailure' empty_volume = Volume(name='empty-volume', empty_dir={}) volume_mount = VolumeMount(name='empty-volume', mount_path='/mnt/mydir') poller = self.client.container_groups.create_or_update( resource_group.name, container_group_name, { 'location': location, 'containers': [{ 'name': container_group_name, 'image': image, 'resources': { 'requests': { 'memory_in_gb': memory, 'cpu': cpu } }, 'volume_mounts': [volume_mount] }], 'os_type': os_type, 'restart_policy': restart_policy, 'volumes': [empty_volume] }) container_group = poller.result() self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) container_group = self.client.container_groups.get( resource_group.name, container_group_name) self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) container_groups = list( self.client.container_groups.list_by_resource_group( resource_group.name)) self.assertEqual(len(container_groups), 1) self.assertEqual(container_groups[0].name, container_group_name) self.assertEqual(container_groups[0].location, location) self.assertEqual(container_groups[0].os_type, os_type) self.assertEqual(container_groups[0].restart_policy, restart_policy) self.assertEqual(container_groups[0].containers[0].name, container_group_name) self.assertEqual(container_groups[0].containers[0].image, image) self.assertEqual( container_groups[0].containers[0].resources.requests.memory_in_gb, memory) self.assertEqual( container_groups[0].containers[0].resources.requests.cpu, cpu) self.assertEqual(container_groups[0].volumes[0].name, empty_volume.name) terminal_size = {"rows": 24, "cols": 80} command = "/bin/bash" containerExecResponse = self.client.start_container.launch_exec( resource_group.name, container_group.name, container_group.containers[0].name, command, terminal_size) self.assertNotEqual(containerExecResponse.web_socket_uri, None) self.assertNotEqual(containerExecResponse.password, None)
def execute(self, context): ci_hook = AzureContainerInstanceHook(self.ci_conn_id) if self.fail_if_exists: self.log.info("Testing if container group already exists") if ci_hook.exists(self.resource_group, self.name): raise AirflowException("Container group exists") if self.registry_conn_id: registry_hook = AzureContainerRegistryHook(self.registry_conn_id) image_registry_credentials = [ registry_hook.connection, ] else: image_registry_credentials = None environment_variables = [] for key, value in self.environment_variables.items(): environment_variables.append(EnvironmentVariable(key, value)) volumes = [] volume_mounts = [] for conn_id, account_name, share_name, mount_path, read_only in self.volumes: hook = AzureContainerVolumeHook(conn_id) mount_name = "mount-%d" % len(volumes) volumes.append( hook.get_file_volume(mount_name, share_name, account_name, read_only)) volume_mounts.append(VolumeMount(mount_name, mount_path, read_only)) exit_code = 1 try: self.log.info("Starting container group with %.1f cpu %.1f mem", self.cpu, self.memory_in_gb) resources = ResourceRequirements(requests=ResourceRequests( memory_in_gb=self.memory_in_gb, cpu=self.cpu)) container = Container(name=self.name, image=self.image, resources=resources, command=self.command, environment_variables=environment_variables, volume_mounts=volume_mounts) container_group = ContainerGroup( location=self.region, containers=[ container, ], image_registry_credentials=image_registry_credentials, volumes=volumes, restart_policy='Never', os_type='Linux') ci_hook.create_or_update(self.resource_group, self.name, container_group) self.log.info("Container group started %s/%s", self.resource_group, self.name) exit_code = self._monitor_logging(ci_hook, self.resource_group, self.name) self.log.info("Container had exit code: %s", exit_code) if exit_code != 0: raise AirflowException( "Container had a non-zero exit code, %s" % exit_code) except CloudError: self.log.exception("Could not start container group") raise AirflowException("Could not start container group") finally: if exit_code == 0 or self.remove_on_error: self.log.info("Deleting container group") try: ci_hook.delete(self.resource_group, self.name) except Exception: self.log.exception("Could not delete container group")
def test_container_instance(self, resource_group, location): container_group_name = self.get_resource_name('pycontainer') image = 'alpine:latest' os_type = 'Linux' cpu = 1 memory = 1 restart_policy = 'OnFailure' livenessprob_period_seconds = 5 log_analytics_workspace_id = 'workspaceId' log_analytics_workspace_key = 'workspaceKey' empty_volume = Volume(name='empty-volume', empty_dir={}) volume_mount = VolumeMount(name='empty-volume', mount_path='/mnt/mydir') poller = self.client.container_groups.create_or_update( resource_group.name, container_group_name, { 'location': location, 'containers': [{ 'name': container_group_name, 'image': image, 'resources': { 'requests': { 'memory_in_gb': memory, 'cpu': cpu } }, 'volume_mounts': [volume_mount], 'liveness_probe': { 'exec': { 'command': ['cat' '/tmp/healthy'] }, 'periodSeconds': livenessprob_period_seconds } }], 'os_type': os_type, 'restart_policy': restart_policy, 'diagnostics': { 'log_analytics': { 'workspace_id': log_analytics_workspace_id, 'workspace_key': log_analytics_workspace_key } }, 'volumes': [empty_volume], }) container_group = poller.result() self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual( container_group.diagnostics.log_analytics.workspace_id, log_analytics_workspace_id) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) self.assertEqual( container_group.containers[0].liveness_probe.period_seconds, livenessprob_period_seconds) container_group = self.client.container_groups.get( resource_group.name, container_group_name) self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual( container_group.diagnostics.log_analytics.workspace_id, log_analytics_workspace_id) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) self.assertEqual( container_group.containers[0].liveness_probe.period_seconds, livenessprob_period_seconds) container_groups = list( self.client.container_groups.list_by_resource_group( resource_group.name)) self.assertEqual(len(container_groups), 1) self.assertEqual(container_groups[0].name, container_group_name) self.assertEqual(container_groups[0].location, location) self.assertEqual(container_groups[0].os_type, os_type) self.assertEqual(container_groups[0].restart_policy, restart_policy) self.assertEqual( container_groups[0].diagnostics.log_analytics.workspace_id, log_analytics_workspace_id) self.assertEqual(container_groups[0].containers[0].name, container_group_name) self.assertEqual(container_groups[0].containers[0].image, image) self.assertEqual( container_groups[0].containers[0].resources.requests.memory_in_gb, memory) self.assertEqual( container_groups[0].containers[0].resources.requests.cpu, cpu) self.assertEqual(container_groups[0].volumes[0].name, empty_volume.name) self.assertEqual( container_groups[0].containers[0].liveness_probe.period_seconds, livenessprob_period_seconds) # Testing Container_Execute_Command terminal_size = {"rows": 24, "cols": 80} command = "/bin/bash" containerExecResponse = self.client.container.execute_command( resource_group.name, container_group.name, container_group.containers[0].name, command, terminal_size) self.assertNotEqual(containerExecResponse.web_socket_uri, None) self.assertNotEqual(containerExecResponse.password, None) # Testing Container_List_Logs containerLogResponse = self.client.container.list_logs( resource_group.name, container_group.name, container_group.containers[0].name) # Testing Restart Container Group poller = self.client.container_groups.restart(resource_group.name, container_group_name) poller.result()
def test_container_instance(self, resource_group, location): container_group_name = self.get_resource_name('pycontainer') image = 'alpine:latest' os_type = 'Linux' cpu = 1 memory = 1 restart_policy = 'OnFailure' empty_volume = Volume(name='empty-volume', empty_dir={}) volume_mount = VolumeMount(name='empty-volume', mount_path='/mnt/mydir') container_group = self.client.container_groups.create_or_update( resource_group.name, container_group_name, { 'location': location, 'containers': [{ 'name': container_group_name, 'image': image, 'resources': { 'requests': { 'memory_in_gb': memory, 'cpu': cpu } }, 'volume_mounts': [volume_mount] }], 'os_type': os_type, 'restart_policy': restart_policy, 'volumes': [empty_volume] }) self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) container_group = self.client.container_groups.get( resource_group.name, container_group_name) self.assertEqual(container_group.name, container_group_name) self.assertEqual(container_group.location, location) self.assertEqual(container_group.os_type, os_type) self.assertEqual(container_group.restart_policy, restart_policy) self.assertEqual(container_group.containers[0].name, container_group_name) self.assertEqual(container_group.containers[0].image, image) self.assertEqual( container_group.containers[0].resources.requests.memory_in_gb, memory) self.assertEqual(container_group.containers[0].resources.requests.cpu, cpu) self.assertEqual(container_group.volumes[0].name, empty_volume.name) container_groups = list( self.client.container_groups.list_by_resource_group( resource_group.name)) self.assertEqual(len(container_groups), 1) self.assertEqual(container_groups[0].name, container_group_name) self.assertEqual(container_groups[0].location, location) self.assertEqual(container_groups[0].os_type, os_type) self.assertEqual(container_groups[0].restart_policy, restart_policy) self.assertEqual(container_groups[0].containers[0].name, container_group_name) self.assertEqual(container_groups[0].containers[0].image, image) self.assertEqual( container_groups[0].containers[0].resources.requests.memory_in_gb, memory) self.assertEqual( container_groups[0].containers[0].resources.requests.cpu, cpu) self.assertEqual(container_groups[0].volumes[0].name, empty_volume.name)