def test_get_file_volume(self):
db.merge_conn(
Connection(conn_id='wasb_test_key',
conn_type='wasb',
login='******',
password='******'))
hook = AzureContainerVolumeHook(wasb_conn_id='wasb_test_key')
volume = hook.get_file_volume(mount_name='mount',
share_name='share',
storage_account_name='storage',
read_only=True)
self.assertIsNotNone(volume)
self.assertEqual(volume.name, 'mount')
self.assertEqual(volume.azure_file.share_name, 'share')
self.assertEqual(volume.azure_file.storage_account_key, 'key')
self.assertEqual(volume.azure_file.storage_account_name, 'storage')
self.assertEqual(volume.azure_file.read_only, True)
def test_get_file_volume(self):
configuration.load_test_config()
db.merge_conn(
Connection(
conn_id='wasb_test_key',
conn_type='wasb',
login='******',
password='******'
)
)
hook = AzureContainerVolumeHook(wasb_conn_id='wasb_test_key')
volume = hook.get_file_volume(mount_name='mount',
share_name='share',
storage_account_name='storage',
read_only=True)
self.assertIsNotNone(volume)
self.assertEqual(volume.name, 'mount')
self.assertEqual(volume.azure_file.share_name, 'share')
self.assertEqual(volume.azure_file.storage_account_key, 'key')
self.assertEqual(volume.azure_file.storage_account_name, 'storage')
self.assertEqual(volume.azure_file.read_only, True)
def execute(self, context):
ci_hook = AzureContainerInstanceHook(self.ci_conn_id)
if self.fail_if_exists:
self.log.info("Testing if container group already exists")
if ci_hook.exists(self.resource_group, self.name):
raise AirflowException("Container group exists")
if self.registry_conn_id:
registry_hook = AzureContainerRegistryHook(self.registry_conn_id)
image_registry_credentials = [registry_hook.connection, ]
else:
image_registry_credentials = None
environment_variables = []
for key, value in self.environment_variables.items():
environment_variables.append(EnvironmentVariable(key, value))
volumes = []
volume_mounts = []
for conn_id, account_name, share_name, mount_path, read_only in self.volumes:
hook = AzureContainerVolumeHook(conn_id)
mount_name = "mount-%d" % len(volumes)
volumes.append(hook.get_file_volume(mount_name,
share_name,
account_name,
read_only))
volume_mounts.append(VolumeMount(mount_name, mount_path, read_only))
exit_code = 1
try:
self.log.info("Starting container group with %.1f cpu %.1f mem",
self.cpu, self.memory_in_gb)
resources = ResourceRequirements(requests=ResourceRequests(
memory_in_gb=self.memory_in_gb,
cpu=self.cpu))
container = Container(
name=self.name,
image=self.image,
resources=resources,
command=self.command,
environment_variables=environment_variables,
volume_mounts=volume_mounts)
container_group = ContainerGroup(
location=self.region,
containers=[container, ],
image_registry_credentials=image_registry_credentials,
volumes=volumes,
restart_policy='Never',
os_type='Linux')
ci_hook.create_or_update(self.resource_group, self.name, container_group)
self.log.info("Container group started %s/%s", self.resource_group, self.name)
exit_code = self._monitor_logging(ci_hook, self.resource_group, self.name)
self.log.info("Container had exit code: %s", exit_code)
if exit_code != 0:
raise AirflowException("Container had a non-zero exit code, %s"
% exit_code)
except CloudError:
self.log.exception("Could not start container group")
raise AirflowException("Could not start container group")
finally:
if exit_code == 0 or self.remove_on_error:
self.log.info("Deleting container group")
try:
ci_hook.delete(self.resource_group, self.name)
except Exception:
self.log.exception("Could not delete container group")
def execute(self, context):
# Check name again in case it was templated.
self._check_name(self.name)
self._ci_hook = AzureContainerInstanceHook(self.ci_conn_id)
if self.fail_if_exists:
self.log.info("Testing if container group already exists")
if self._ci_hook.exists(self.resource_group, self.name):
raise AirflowException("Container group exists")
if self.registry_conn_id:
registry_hook = AzureContainerRegistryHook(self.registry_conn_id)
image_registry_credentials = [
registry_hook.connection,
]
else:
image_registry_credentials = None
environment_variables = []
for key, value in self.environment_variables.items():
if key in self.secured_variables:
e = EnvironmentVariable(name=key, secure_value=value)
else:
e = EnvironmentVariable(name=key, value=value)
environment_variables.append(e)
volumes = []
volume_mounts = []
for conn_id, account_name, share_name, mount_path, read_only in self.volumes:
hook = AzureContainerVolumeHook(conn_id)
mount_name = "mount-%d" % len(volumes)
volumes.append(
hook.get_file_volume(mount_name, share_name, account_name,
read_only))
volume_mounts.append(
VolumeMount(name=mount_name,
mount_path=mount_path,
read_only=read_only))
exit_code = 1
try:
self.log.info("Starting container group with %.1f cpu %.1f mem",
self.cpu, self.memory_in_gb)
if self.gpu:
self.log.info("GPU count: %.1f, GPU SKU: %s", self.gpu.count,
self.gpu.sku)
resources = ResourceRequirements(requests=ResourceRequests(
memory_in_gb=self.memory_in_gb, cpu=self.cpu, gpu=self.gpu))
container = Container(name=self.name,
image=self.image,
resources=resources,
command=self.command,
environment_variables=environment_variables,
volume_mounts=volume_mounts)
container_group = ContainerGroup(
location=self.region,
containers=[
container,
],
image_registry_credentials=image_registry_credentials,
volumes=volumes,
restart_policy='Never',
os_type='Linux')
self._ci_hook.create_or_update(self.resource_group, self.name,
container_group)
self.log.info("Container group started %s/%s", self.resource_group,
self.name)
exit_code = self._monitor_logging(self._ci_hook,
self.resource_group, self.name)
self.log.info("Container had exit code: %s", exit_code)
if exit_code != 0:
raise AirflowException(
"Container had a non-zero exit code, %s" % exit_code)
return exit_code
except CloudError:
self.log.exception("Could not start container group")
raise AirflowException("Could not start container group")
finally:
if exit_code == 0 or self.remove_on_error:
self.on_kill()