def _create_container_group(self, resource_group_name,
name,
location,
image,
memory,
cpu,
algorithm,
asset,
input_mount_point,
output_moint_point,
account_name,
account_key,
share_name_input,
share_name_output):
# setup default values
result_file = 'result-' + str(int(time.time()))
command = ['python', input_mount_point + '/' + algorithm, input_mount_point + '/' + asset,
output_moint_point + '/' + result_file]
environment_variables = None
az_file_input = AzureFileVolume(share_name=share_name_input,
storage_account_name=account_name,
storage_account_key=account_key,
)
az_file_output = AzureFileVolume(share_name=share_name_output,
storage_account_name=account_name,
storage_account_key=account_key,
)
volume = [Volume(name=share_name_input, azure_file=az_file_input),
Volume(name=share_name_output, azure_file=az_file_output)]
volume_mount = [VolumeMount(name=share_name_input, mount_path=input_mount_point),
VolumeMount(name=share_name_output, mount_path=output_moint_point)]
# set memory and cpu
container_resource_requests = ResourceRequests(memory_in_gb=memory, cpu=cpu)
container_resource_requirements = ResourceRequirements(requests=container_resource_requests)
container = Container(name=name,
image=image,
resources=container_resource_requirements,
command=command,
environment_variables=environment_variables,
volume_mounts=volume_mount,
)
# defaults for container group
cgroup_os_type = OperatingSystemTypes.linux
cgroup = ContainerGroup(location=location,
containers=[container],
os_type=cgroup_os_type,
restart_policy=ContainerGroupRestartPolicy.never,
volumes=volume,
)
self.client.container_groups.create_or_update(resource_group_name, name, cgroup)
return result_file
def create_container_group(az_conf, client, resource_group_name, name, location, image, memory, cpu):
# Start new containers
# setup default values
port = 80
container_resource_requirements = None
command = None
environment_variables = None
volume_mount = [VolumeMount(name='config', mount_path='/mnt/conf'),VolumeMount(name='data', mount_path='/mnt/data')]
# data_volume_mount = [VolumeMount(name='data', mount_path='/mnt/data')]
# set memory and cpu
container_resource_requests = ResourceRequests(memory_in_gb = memory, cpu = cpu)
container_resource_requirements = ResourceRequirements(requests = container_resource_requests)
az_config_file_volume = AzureFileVolume(share_name='config',
storage_account_name=az_conf['storage_account_name'],
storage_account_key=az_conf['storage_account_key'])
az_data_file_volume = AzureFileVolume(share_name='data',
storage_account_name=az_conf['storage_account_name'],
storage_account_key=az_conf['storage_account_key'])
#volume_mount = VolumeMount(name='config', mount_path='/mnt/config')
volumes = [Volume(name='config', azure_file=az_config_file_volume),Volume(name='data', azure_file=az_data_file_volume)]
container = Container(name = name,
image = image,
resources = container_resource_requirements,
command = command,
ports = [ContainerPort(port=port)],
environment_variables = environment_variables,
volume_mounts = volume_mount)
# defaults for container group
cgroup_os_type = OperatingSystemTypes.linux
cgroup_restart_policy = ContainerGroupRestartPolicy.never
image_registry_credential = ImageRegistryCredential(server=az_conf['container_registry'],
username=az_conf['container_registry_user'],
password=az_conf['container_registry_pwd'])
cgroup = ContainerGroup(location = location,
containers = [container],
os_type = cgroup_os_type,
image_registry_credentials = [image_registry_credential],
restart_policy = cgroup_restart_policy,
volumes = volumes)
toto = client.container_groups.create_or_update(resource_group_name, name, cgroup)
#print(toto.containers)
"""
def _create_gitrepo_volume_mount(gitrepo_volume, gitrepo_mount_path):
"""Create Git Repo volume mount. """
if gitrepo_mount_path:
if not gitrepo_volume:
raise CLIError('Please specify --gitrepo-url (--gitrepo-dir --gitrepo-revision) '
'to enable Git Repo volume mount.')
return VolumeMount(name=GITREPO_VOLUME_NAME, mount_path=gitrepo_mount_path)
def _create_secrets_volume_mount(secrets_volume, secrets_mount_path):
"""Create secrets volume mount. """
if secrets_volume:
if not secrets_mount_path:
raise CLIError('Please specify --secrets --secrets-mount-path '
'to enable secrets volume mount.')
return VolumeMount(name=SECRETS_VOLUME_NAME, mount_path=secrets_mount_path)
def _create_azure_file_volume_mount(azure_file_volume, azure_file_volume_mount_path):
"""Create Azure File volume mount. """
if azure_file_volume_mount_path:
if not azure_file_volume:
raise CLIError('Please specify --azure-file-volume-share-name --azure-file-volume-account-name --azure-file-volume-account-key '
'to enable Azure File volume mount.')
return VolumeMount(name=AZURE_FILE_VOLUME_NAME, mount_path=azure_file_volume_mount_path)
def create_container(name, image, pr_number, user_id, resource_group, port=None, location='westus'):
from azure.mgmt.containerinstance.models import (ContainerGroup, Container, ContainerPort, Port, IpAddress,
ImageRegistryCredential, ResourceRequirements, ResourceRequests,
ContainerGroupNetworkProtocol, OperatingSystemTypes, EnvironmentVariable,
Volume, AzureFileVolume, VolumeMount)
from random import randint
import secrets
pr_number = str(pr_number)
if port is None:
port = randint(1000, 65000)
instance_token = secrets.token_urlsafe(256)
environment_variables = [EnvironmentVariable('PR_NUM', pr_number),
EnvironmentVariable('PORT', port),
EnvironmentVariable('INSTANCE_TOKEN', instance_token)]
tags = {'userId': user_id, 'prNumber': pr_number}
container_resource_requests = ResourceRequests(memory_in_gb='1.5', cpu='1')
container_resource_requirements = ResourceRequirements(requests=container_resource_requests)
container = Container(name=name,
image=image,
resources=container_resource_requirements,
ports=[ContainerPort(port=port)],
environment_variables=environment_variables,
volume_mounts=[VolumeMount('volume1', '/cert', read_only=True)])
cgroup_ip_address = IpAddress(ports=[Port(protocol=ContainerGroupNetworkProtocol.tcp, port=port)])
cgroup_os_type = OperatingSystemTypes.linux
afv = AzureFileVolume(SHARE_NAME, SHARE_STORAGE_ACCOUNT_NAME, read_only=True, storage_account_key=SHARE_STORAGE_ACCOUNT_KEY)
cgroup = ContainerGroup(location=location,
containers=[container],
os_type=cgroup_os_type,
ip_address=cgroup_ip_address,
tags=tags,
volumes=[Volume('volume1', afv)])
return get_aci_client().container_groups.create_or_update(resource_group, name, cgroup)
def _create_container_group(client, resource_group_name, name, location, image,
memory, cpu, volumes):
# Start new containers
# setup default values
port = 80
container_resource_requirements = None
command = None
environment_variables = None
if volumes:
az_volume_mount = []
az_volumes = []
for volume in volumes:
az_volume_mount = az_volume_mount + [
VolumeMount(name=str(volume['name']),
mount_path=volume['mount_path'])
]
az_file = AzureFileVolume(
share_name=str(volume['name']),
storage_account_name=Variable.get("STORAGE_ACCOUNT_NAME"),
storage_account_key=Variable.get("STORAGE_ACCOUNT_KEY_PASSWD"))
az_volumes = az_volumes + [
Volume(name=str(volume['name']), azure_file=az_file)
]
# set memory and cpu
container_resource_requests = ResourceRequests(memory_in_gb=memory,
cpu=cpu)
container_resource_requirements = ResourceRequirements(
requests=container_resource_requests)
container = Container(name=name,
image=image,
resources=container_resource_requirements,
command=command,
ports=[ContainerPort(port=port)],
environment_variables=environment_variables,
volume_mounts=az_volume_mount)
# defaults for container group
cgroup_os_type = OperatingSystemTypes.linux
cgroup_restart_policy = ContainerGroupRestartPolicy.never
image_registry_credential = ImageRegistryCredential(
server=Variable.get("CONTAINER_REGISTRY"),
username=Variable.get("CONTAINER_REGISTRY_USER"),
password=Variable.get("CONTAINER_REGISTRY_PASSWD"))
cgroup = ContainerGroup(
location=location,
containers=[container],
os_type=cgroup_os_type,
image_registry_credentials=[image_registry_credential],
restart_policy=cgroup_restart_policy,
volumes=az_volumes)
results = client.container_groups.create_or_update(resource_group_name,
name, cgroup)
return (results)
def prepare_azure_volumes(self, afs_volumes: list = []):
volumes = []
volume_mounts = []
for afs in afs_volumes:
assert afs['afs_name'] is not None
assert afs['afs_key'] is not None
assert afs['afs_share'] is not None
assert afs['mount_path'] is not None
assert afs['name'] is not None
az_volume = AzureFileVolume(share_name=afs['afs_share'],
storage_account_name=afs['afs_name'],
storage_account_key=afs['afs_key'])
volumes.append(Volume(name=afs['name'], azure_file=az_volume))
volume_mounts.append(
VolumeMount(name=afs['name'], mount_path=afs['mount_path']))
return volumes, volume_mounts
def execute(self, context: dict) -> int:
# Check name again in case it was templated.
self._check_name(self.name)
self._ci_hook = AzureContainerInstanceHook(self.ci_conn_id)
if self.fail_if_exists:
self.log.info("Testing if container group already exists")
if self._ci_hook.exists(self.resource_group, self.name):
raise AirflowException("Container group exists")
if self.registry_conn_id:
registry_hook = AzureContainerRegistryHook(self.registry_conn_id)
image_registry_credentials: Optional[list] = [
registry_hook.connection,
]
else:
image_registry_credentials = None
environment_variables = []
for key, value in self.environment_variables.items():
if key in self.secured_variables:
e = EnvironmentVariable(name=key, secure_value=value)
else:
e = EnvironmentVariable(name=key, value=value)
environment_variables.append(e)
volumes: List[Union[Volume, Volume]] = []
volume_mounts: List[Union[VolumeMount, VolumeMount]] = []
for conn_id, account_name, share_name, mount_path, read_only in self.volumes:
hook = AzureContainerVolumeHook(conn_id)
mount_name = "mount-%d" % len(volumes)
volumes.append(
hook.get_file_volume(mount_name, share_name, account_name,
read_only))
volume_mounts.append(
VolumeMount(name=mount_name,
mount_path=mount_path,
read_only=read_only))
exit_code = 1
try:
self.log.info("Starting container group with %.1f cpu %.1f mem",
self.cpu, self.memory_in_gb)
if self.gpu:
self.log.info("GPU count: %.1f, GPU SKU: %s", self.gpu.count,
self.gpu.sku)
resources = ResourceRequirements(requests=ResourceRequests(
memory_in_gb=self.memory_in_gb, cpu=self.cpu, gpu=self.gpu))
if self.ip_address and not self.ports:
self.ports = [ContainerPort(port=80)]
self.log.info(
"Default port set. Container will listen on port 80")
container = Container(
name=self.name,
image=self.image,
resources=resources,
command=self.command,
environment_variables=environment_variables,
volume_mounts=volume_mounts,
ports=self.ports,
)
container_group = ContainerGroup(
location=self.region,
containers=[
container,
],
image_registry_credentials=image_registry_credentials,
volumes=volumes,
restart_policy=self.restart_policy,
os_type=self.os_type,
tags=self.tags,
ip_address=self.ip_address,
)
self._ci_hook.create_or_update(self.resource_group, self.name,
container_group)
self.log.info("Container group started %s/%s", self.resource_group,
self.name)
exit_code = self._monitor_logging(self.resource_group, self.name)
self.log.info("Container had exit code: %s", exit_code)
if exit_code != 0:
raise AirflowException(
f"Container had a non-zero exit code, {exit_code}")
return exit_code
except CloudError:
self.log.exception("Could not start container group")
raise AirflowException("Could not start container group")
finally:
if exit_code == 0 or self.remove_on_error:
self.on_kill()
def test_container_instance(self, resource_group, location):
container_group_name = self.get_resource_name('pycontainer')
image = 'alpine:latest'
os_type = 'Linux'
cpu = 1
memory = 1
restart_policy = 'OnFailure'
empty_volume = Volume(name='empty-volume', empty_dir={})
volume_mount = VolumeMount(name='empty-volume',
mount_path='/mnt/mydir')
poller = self.client.container_groups.create_or_update(
resource_group.name, container_group_name, {
'location':
location,
'containers': [{
'name': container_group_name,
'image': image,
'resources': {
'requests': {
'memory_in_gb': memory,
'cpu': cpu
}
},
'volume_mounts': [volume_mount]
}],
'os_type':
os_type,
'restart_policy':
restart_policy,
'volumes': [empty_volume]
})
container_group = poller.result()
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
container_group = self.client.container_groups.get(
resource_group.name, container_group_name)
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
container_groups = list(
self.client.container_groups.list_by_resource_group(
resource_group.name))
self.assertEqual(len(container_groups), 1)
self.assertEqual(container_groups[0].name, container_group_name)
self.assertEqual(container_groups[0].location, location)
self.assertEqual(container_groups[0].os_type, os_type)
self.assertEqual(container_groups[0].restart_policy, restart_policy)
self.assertEqual(container_groups[0].containers[0].name,
container_group_name)
self.assertEqual(container_groups[0].containers[0].image, image)
self.assertEqual(
container_groups[0].containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(
container_groups[0].containers[0].resources.requests.cpu, cpu)
self.assertEqual(container_groups[0].volumes[0].name,
empty_volume.name)
terminal_size = {"rows": 24, "cols": 80}
command = "/bin/bash"
containerExecResponse = self.client.start_container.launch_exec(
resource_group.name, container_group.name,
container_group.containers[0].name, command, terminal_size)
self.assertNotEqual(containerExecResponse.web_socket_uri, None)
self.assertNotEqual(containerExecResponse.password, None)
def execute(self, context):
ci_hook = AzureContainerInstanceHook(self.ci_conn_id)
if self.fail_if_exists:
self.log.info("Testing if container group already exists")
if ci_hook.exists(self.resource_group, self.name):
raise AirflowException("Container group exists")
if self.registry_conn_id:
registry_hook = AzureContainerRegistryHook(self.registry_conn_id)
image_registry_credentials = [
registry_hook.connection,
]
else:
image_registry_credentials = None
environment_variables = []
for key, value in self.environment_variables.items():
environment_variables.append(EnvironmentVariable(key, value))
volumes = []
volume_mounts = []
for conn_id, account_name, share_name, mount_path, read_only in self.volumes:
hook = AzureContainerVolumeHook(conn_id)
mount_name = "mount-%d" % len(volumes)
volumes.append(
hook.get_file_volume(mount_name, share_name, account_name,
read_only))
volume_mounts.append(VolumeMount(mount_name, mount_path,
read_only))
exit_code = 1
try:
self.log.info("Starting container group with %.1f cpu %.1f mem",
self.cpu, self.memory_in_gb)
resources = ResourceRequirements(requests=ResourceRequests(
memory_in_gb=self.memory_in_gb, cpu=self.cpu))
container = Container(name=self.name,
image=self.image,
resources=resources,
command=self.command,
environment_variables=environment_variables,
volume_mounts=volume_mounts)
container_group = ContainerGroup(
location=self.region,
containers=[
container,
],
image_registry_credentials=image_registry_credentials,
volumes=volumes,
restart_policy='Never',
os_type='Linux')
ci_hook.create_or_update(self.resource_group, self.name,
container_group)
self.log.info("Container group started %s/%s", self.resource_group,
self.name)
exit_code = self._monitor_logging(ci_hook, self.resource_group,
self.name)
self.log.info("Container had exit code: %s", exit_code)
if exit_code != 0:
raise AirflowException(
"Container had a non-zero exit code, %s" % exit_code)
except CloudError:
self.log.exception("Could not start container group")
raise AirflowException("Could not start container group")
finally:
if exit_code == 0 or self.remove_on_error:
self.log.info("Deleting container group")
try:
ci_hook.delete(self.resource_group, self.name)
except Exception:
self.log.exception("Could not delete container group")
def test_container_instance(self, resource_group, location):
container_group_name = self.get_resource_name('pycontainer')
image = 'alpine:latest'
os_type = 'Linux'
cpu = 1
memory = 1
restart_policy = 'OnFailure'
livenessprob_period_seconds = 5
log_analytics_workspace_id = 'workspaceId'
log_analytics_workspace_key = 'workspaceKey'
empty_volume = Volume(name='empty-volume', empty_dir={})
volume_mount = VolumeMount(name='empty-volume',
mount_path='/mnt/mydir')
poller = self.client.container_groups.create_or_update(
resource_group.name, container_group_name, {
'location':
location,
'containers': [{
'name': container_group_name,
'image': image,
'resources': {
'requests': {
'memory_in_gb': memory,
'cpu': cpu
}
},
'volume_mounts': [volume_mount],
'liveness_probe': {
'exec': {
'command': ['cat'
'/tmp/healthy']
},
'periodSeconds': livenessprob_period_seconds
}
}],
'os_type':
os_type,
'restart_policy':
restart_policy,
'diagnostics': {
'log_analytics': {
'workspace_id': log_analytics_workspace_id,
'workspace_key': log_analytics_workspace_key
}
},
'volumes': [empty_volume],
})
container_group = poller.result()
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(
container_group.diagnostics.log_analytics.workspace_id,
log_analytics_workspace_id)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
self.assertEqual(
container_group.containers[0].liveness_probe.period_seconds,
livenessprob_period_seconds)
container_group = self.client.container_groups.get(
resource_group.name, container_group_name)
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(
container_group.diagnostics.log_analytics.workspace_id,
log_analytics_workspace_id)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
self.assertEqual(
container_group.containers[0].liveness_probe.period_seconds,
livenessprob_period_seconds)
container_groups = list(
self.client.container_groups.list_by_resource_group(
resource_group.name))
self.assertEqual(len(container_groups), 1)
self.assertEqual(container_groups[0].name, container_group_name)
self.assertEqual(container_groups[0].location, location)
self.assertEqual(container_groups[0].os_type, os_type)
self.assertEqual(container_groups[0].restart_policy, restart_policy)
self.assertEqual(
container_groups[0].diagnostics.log_analytics.workspace_id,
log_analytics_workspace_id)
self.assertEqual(container_groups[0].containers[0].name,
container_group_name)
self.assertEqual(container_groups[0].containers[0].image, image)
self.assertEqual(
container_groups[0].containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(
container_groups[0].containers[0].resources.requests.cpu, cpu)
self.assertEqual(container_groups[0].volumes[0].name,
empty_volume.name)
self.assertEqual(
container_groups[0].containers[0].liveness_probe.period_seconds,
livenessprob_period_seconds)
# Testing Container_Execute_Command
terminal_size = {"rows": 24, "cols": 80}
command = "/bin/bash"
containerExecResponse = self.client.container.execute_command(
resource_group.name, container_group.name,
container_group.containers[0].name, command, terminal_size)
self.assertNotEqual(containerExecResponse.web_socket_uri, None)
self.assertNotEqual(containerExecResponse.password, None)
# Testing Container_List_Logs
containerLogResponse = self.client.container.list_logs(
resource_group.name, container_group.name,
container_group.containers[0].name)
# Testing Restart Container Group
poller = self.client.container_groups.restart(resource_group.name,
container_group_name)
poller.result()
def test_container_instance(self, resource_group, location):
container_group_name = self.get_resource_name('pycontainer')
image = 'alpine:latest'
os_type = 'Linux'
cpu = 1
memory = 1
restart_policy = 'OnFailure'
empty_volume = Volume(name='empty-volume', empty_dir={})
volume_mount = VolumeMount(name='empty-volume',
mount_path='/mnt/mydir')
container_group = self.client.container_groups.create_or_update(
resource_group.name, container_group_name, {
'location':
location,
'containers': [{
'name': container_group_name,
'image': image,
'resources': {
'requests': {
'memory_in_gb': memory,
'cpu': cpu
}
},
'volume_mounts': [volume_mount]
}],
'os_type':
os_type,
'restart_policy':
restart_policy,
'volumes': [empty_volume]
})
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
container_group = self.client.container_groups.get(
resource_group.name, container_group_name)
self.assertEqual(container_group.name, container_group_name)
self.assertEqual(container_group.location, location)
self.assertEqual(container_group.os_type, os_type)
self.assertEqual(container_group.restart_policy, restart_policy)
self.assertEqual(container_group.containers[0].name,
container_group_name)
self.assertEqual(container_group.containers[0].image, image)
self.assertEqual(
container_group.containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(container_group.containers[0].resources.requests.cpu,
cpu)
self.assertEqual(container_group.volumes[0].name, empty_volume.name)
container_groups = list(
self.client.container_groups.list_by_resource_group(
resource_group.name))
self.assertEqual(len(container_groups), 1)
self.assertEqual(container_groups[0].name, container_group_name)
self.assertEqual(container_groups[0].location, location)
self.assertEqual(container_groups[0].os_type, os_type)
self.assertEqual(container_groups[0].restart_policy, restart_policy)
self.assertEqual(container_groups[0].containers[0].name,
container_group_name)
self.assertEqual(container_groups[0].containers[0].image, image)
self.assertEqual(
container_groups[0].containers[0].resources.requests.memory_in_gb,
memory)
self.assertEqual(
container_groups[0].containers[0].resources.requests.cpu, cpu)
self.assertEqual(container_groups[0].volumes[0].name,
empty_volume.name)
