def create_image_registry_credentials(registry_login_server, registry_username, registry_password, image):
"""Create image registry credentials. """
image_registry_credentials = None
if registry_login_server:
if not registry_username:
raise CLIError('Please specify --registry-username in order to use custom image registry.')
if not registry_password:
try:
registry_password = prompt_pass(msg='Image registry password: '******'Please specify --registry-password in order to use custom image registry.')
image_registry_credentials = [ImageRegistryCredential(server=registry_login_server,
username=registry_username,
password=registry_password)]
elif ACR_SERVER_SUFFIX in image:
if not registry_password:
try:
registry_password = prompt_pass(msg='Image registry password: '******'Please specify --registry-password in order to use Azure Container Registry.')
acr_server = image.split("/")[0] if image.split("/") else None
acr_username = image.split(ACR_SERVER_SUFFIX)[0] if image.split(ACR_SERVER_SUFFIX) else None
if acr_server and acr_username:
image_registry_credentials = [ImageRegistryCredential(server=acr_server,
username=acr_username,
password=registry_password)]
else:
raise CLIError('Failed to parse ACR server or username from image name; please explicitly specify --registry-server and --registry-username.')
return image_registry_credentials
def _create_image_registry_credentials(cmd, resource_group_name, registry_login_server, registry_username, registry_password, image, identity):
from msrestazure.tools import is_valid_resource_id
image_registry_credentials = None
if registry_login_server:
if not registry_username:
raise RequiredArgumentMissingError('Please specify --registry-username in order to use custom image registry.')
if not registry_password:
try:
registry_password = prompt_pass(msg='Image registry password: '******'Please specify --registry-password in order to use custom image registry.')
image_registry_credentials = [ImageRegistryCredential(server=registry_login_server,
username=registry_username,
password=registry_password)]
elif ACR_SERVER_DELIMITER in image.split("/")[0]:
acr_server = image.split("/")[0] if image.split("/") else None
if identity:
if not is_valid_resource_id(identity):
msi_client = cf_msi(cmd.cli_ctx)
identity = msi_client.user_assigned_identities.get(resource_group_name=resource_group_name,
resource_name=identity).id
if acr_server:
image_registry_credentials = [ImageRegistryCredential(server=acr_server,
username=registry_username,
password=registry_password,
identity=identity)]
else:
if not registry_username:
try:
registry_username = prompt(msg='Image registry username: '******'Please specify --registry-username in order to use Azure Container Registry.')
if not registry_password:
try:
registry_password = prompt_pass(msg='Image registry password: '******'Please specify --registry-password in order to use Azure Container Registry.')
if acr_server:
image_registry_credentials = [ImageRegistryCredential(server=acr_server,
username=registry_username,
password=registry_password)]
elif registry_username and registry_password and SERVER_DELIMITER in image.split("/")[0]:
login_server = image.split("/")[0] if image.split("/") else None
if login_server:
image_registry_credentials = [ImageRegistryCredential(server=login_server,
username=registry_username,
password=registry_password)]
else:
raise RequiredArgumentMissingError('Failed to parse login server from image name; please explicitly specify --registry-server.')
return image_registry_credentials
def _create_container_group(client, resource_group_name, name, location, image,
memory, cpu, volumes):
# Start new containers
# setup default values
port = 80
container_resource_requirements = None
command = None
environment_variables = None
if volumes:
az_volume_mount = []
az_volumes = []
for volume in volumes:
az_volume_mount = az_volume_mount + [
VolumeMount(name=str(volume['name']),
mount_path=volume['mount_path'])
]
az_file = AzureFileVolume(
share_name=str(volume['name']),
storage_account_name=Variable.get("STORAGE_ACCOUNT_NAME"),
storage_account_key=Variable.get("STORAGE_ACCOUNT_KEY_PASSWD"))
az_volumes = az_volumes + [
Volume(name=str(volume['name']), azure_file=az_file)
]
# set memory and cpu
container_resource_requests = ResourceRequests(memory_in_gb=memory,
cpu=cpu)
container_resource_requirements = ResourceRequirements(
requests=container_resource_requests)
container = Container(name=name,
image=image,
resources=container_resource_requirements,
command=command,
ports=[ContainerPort(port=port)],
environment_variables=environment_variables,
volume_mounts=az_volume_mount)
# defaults for container group
cgroup_os_type = OperatingSystemTypes.linux
cgroup_restart_policy = ContainerGroupRestartPolicy.never
image_registry_credential = ImageRegistryCredential(
server=Variable.get("CONTAINER_REGISTRY"),
username=Variable.get("CONTAINER_REGISTRY_USER"),
password=Variable.get("CONTAINER_REGISTRY_PASSWD"))
cgroup = ContainerGroup(
location=location,
containers=[container],
os_type=cgroup_os_type,
image_registry_credentials=[image_registry_credential],
restart_policy=cgroup_restart_policy,
volumes=az_volumes)
results = client.container_groups.create_or_update(resource_group_name,
name, cgroup)
return (results)
def create_container_group(az_conf, client, resource_group_name, name, location, image, memory, cpu):
# Start new containers
# setup default values
port = 80
container_resource_requirements = None
command = None
environment_variables = None
volume_mount = [VolumeMount(name='config', mount_path='/mnt/conf'),VolumeMount(name='data', mount_path='/mnt/data')]
# data_volume_mount = [VolumeMount(name='data', mount_path='/mnt/data')]
# set memory and cpu
container_resource_requests = ResourceRequests(memory_in_gb = memory, cpu = cpu)
container_resource_requirements = ResourceRequirements(requests = container_resource_requests)
az_config_file_volume = AzureFileVolume(share_name='config',
storage_account_name=az_conf['storage_account_name'],
storage_account_key=az_conf['storage_account_key'])
az_data_file_volume = AzureFileVolume(share_name='data',
storage_account_name=az_conf['storage_account_name'],
storage_account_key=az_conf['storage_account_key'])
#volume_mount = VolumeMount(name='config', mount_path='/mnt/config')
volumes = [Volume(name='config', azure_file=az_config_file_volume),Volume(name='data', azure_file=az_data_file_volume)]
container = Container(name = name,
image = image,
resources = container_resource_requirements,
command = command,
ports = [ContainerPort(port=port)],
environment_variables = environment_variables,
volume_mounts = volume_mount)
# defaults for container group
cgroup_os_type = OperatingSystemTypes.linux
cgroup_restart_policy = ContainerGroupRestartPolicy.never
image_registry_credential = ImageRegistryCredential(server=az_conf['container_registry'],
username=az_conf['container_registry_user'],
password=az_conf['container_registry_pwd'])
cgroup = ContainerGroup(location = location,
containers = [container],
os_type = cgroup_os_type,
image_registry_credentials = [image_registry_credential],
restart_policy = cgroup_restart_policy,
volumes = volumes)
toto = client.container_groups.create_or_update(resource_group_name, name, cgroup)
#print(toto.containers)
"""
def create_container(azure_auth, registry_credentials, container_info,
env_vars):
"""Create a new container group."""
_LOGGER.info("Creating container group '%s'...",
container_info["groupName"])
# Map registry credentials
image_registry_credentials = ImageRegistryCredential(
server=registry_credentials["server"],
username=registry_credentials["username"],
password=registry_credentials["password"],
)
# Map to Azure Container objects
environment_variables = []
for var in env_vars:
environment_variables.append(
EnvironmentVariable(name=var["name"], secure_value=var["value"]))
# Configure the container
container_resource_requests = ResourceRequests(memory_in_gb=1, cpu=1.0)
container_resource_requirements = ResourceRequirements(
requests=container_resource_requests)
container = Container(
name=container_info["groupName"],
image=container_info["image"],
resources=container_resource_requirements,
environment_variables=environment_variables,
)
group = ContainerGroup(
location=container_info["region"],
containers=[container],
os_type=OperatingSystemTypes.linux,
image_registry_credentials=[image_registry_credentials],
restart_policy=ContainerGroupRestartPolicy.never,
)
# Create the container group
aci_client = get_container_client(azure_auth)
aci_client.container_groups.create_or_update(
container_info["resourceGroup"], container_info["groupName"], group)
_LOGGER.info("Created Container group '%s'", container_info["groupName"])
def get_conn(self) -> ImageRegistryCredential:
conn = self.get_connection(self.conn_id)
return ImageRegistryCredential(server=conn.host,
username=conn.login,
password=conn.password)
def create_update_containerinstance(self):
'''
Creates or updates a container service with the specified configuration of orchestrator, masters, and agents.
:return: deserialized container instance state dictionary
'''
self.log("Creating / Updating the container instance {0}".format(
self.name))
registry_credentials = None
if self.registry_login_server is not None:
registry_credentials = [
ImageRegistryCredential(server=self.registry_login_server,
username=self.registry_username,
password=self.registry_password)
]
ip_address = None
if self.ip_address == 'public':
# get list of ports
if self.ports:
ports = []
for port in self.ports:
ports.append(Port(port=port, protocol="TCP"))
ip_address = IpAddress(ports=ports, ip=self.ip_address)
containers = []
for container_def in self.containers:
name = container_def.get("name")
image = container_def.get("image")
memory = container_def.get("memory", 1.5)
cpu = container_def.get("cpu", 1)
ports = []
port_list = container_def.get("ports")
if port_list:
for port in port_list:
ports.append(ContainerPort(port))
containers.append(
Container(name=name,
image=image,
resources=ResourceRequirements(
ResourceRequests(memory_in_gb=memory, cpu=cpu)),
ports=ports))
parameters = ContainerGroup(
location=self.location,
containers=containers,
image_registry_credentials=registry_credentials,
restart_policy=None,
ip_address=ip_address,
os_type=self.os_type,
volumes=None)
response = self.mgmt_client.container_groups.create_or_update(
self.resource_group, self.name, parameters)
return response.as_dict()
def run_task_based_container(self,
container_image_name: str,
command: list = None,
memory_in_gb: int = 1,
cpu: float = 1.0,
gpu_count: int = 0,
gpu_type: str = 'K80',
envs: dict = {},
timeout: int = 600,
afs_volumes: list = [],
volume_mount_path: str = "/input",
afs_name: str = None,
afs_key: str = None,
afs_share: str = None,
afs_mount_subpath: str = '',
image_registry_server: str = None,
image_registry_username: str = None,
image_registry_pwd: str = None,
tag: str = ""):
"""Creates a container group with a single task-based container who's
restart policy is 'Never'. If specified, the container runs a custom
command line at startup.
Arguments:
container_image_name {str}
-- The container image name and tag, for example:
microsoft\aci-helloworld:latest
command {list}
-- The command line that should be executed when the
container starts. This value can be None.
"""
container_group_name = str(id_generator()) + tag
envs['DATA'] = str(Path(volume_mount_path) / afs_mount_subpath)
if command is not None:
cloudhunky_logger.info(
"Creating container group '{0}' with start command '{1}'".
format(container_group_name, command))
gpu = None
if gpu_count > 0:
gpu = GpuResource(count=gpu_count, sku=gpu_type)
container_resource_requests = ResourceRequests(
memory_in_gb=memory_in_gb, cpu=cpu, gpu=gpu)
container_resource_requirements = ResourceRequirements(
requests=container_resource_requests)
environment_variables = []
if envs is not None:
for env, val in envs.items():
environment_variables.append(
EnvironmentVariable(name=env, value=val))
volume_mounts = None
volumes = None
if afs_mount_subpath is not None:
if len(afs_volumes) == 0:
#looks like client using deprecated afs mount method
afs_volumes = [
{
'name': 'azure-volume',
'mount_path': volume_mount_path,
'afs_name': afs_name,
'afs_key': afs_key,
'afs_share': afs_share,
},
]
volumes, volume_mounts = self.prepare_azure_volumes(afs_volumes)
image_registry_credentials = None
if image_registry_username is not None or image_registry_pwd is not None:
if image_registry_username is None:
raise ValueError("insert image_registry_username")
if image_registry_pwd is None:
raise ValueError("insert image_registry_pwd")
if image_registry_server is None:
raise ValueError("insert image_registry_server")
image_registry_credentials = [
ImageRegistryCredential(server=image_registry_server,
username=image_registry_username,
password=image_registry_pwd)
]
container = Container(name=container_group_name,
image=container_image_name,
resources=container_resource_requirements,
command=command,
environment_variables=environment_variables,
volume_mounts=volume_mounts,
ports=None)
group = ContainerGroup(
location=self.resource_group.location,
containers=[container],
os_type=OperatingSystemTypes.linux,
restart_policy=ContainerGroupRestartPolicy.never,
volumes=volumes,
image_registry_credentials=image_registry_credentials)
for i in range(2):
try:
result = self.aci_client.container_groups.create_or_update(
self.resource_group.name, container_group_name, group)
except CloudError as exp:
cloudhunky_logger.exception(exp)
cloudhunky_logger.info("Try to reduce the required resources")
if i == 1:
raise exp
else:
cloudhunky_logger.info("Azure is provisioning container group")
break
# Wait for the container create operation to complete. The operation is
# "done" when the container group provisioning state is one of:
# Succeeded, Canceled, Failed
cloudhunky_logger.info("Container Group is pending")
while result.done() is False:
time.sleep(30)
try:
container_group = self.aci_client.container_groups.get(
self.resource_group.name, container_group_name)
if str(container_group.provisioning_state).lower() == 'succeeded':
cloudhunky_logger.info(
"Creation of container group '{}' succeeded.".format(
container_group_name))
else:
cloudhunky_logger.warning(
"\nCreation of container group '{}' failed. Provisioning state"
"is: {}. Deleting the container group.".format(
container_group_name,
container_group.provisioning_state))
self.aci_client.container_groups.delete(
self.resource_group.name, container_group_name)
return
except Exception as exp:
cloudhunky_logger.exception(exp)
try:
start = time.time()
while timeout > (time.time() - start):
container_group = self.aci_client.container_groups.get(
self.resource_group.name, container_group_name)
container_state = container_group.containers[
0].instance_view.current_state.state
if container_state.lower() == "terminated":
cloudhunky_logger.info("Container terminated")
break
time.sleep(1)
if timeout < (time.time() - start):
cloudhunky_logger.warning(f"Timeout {timeout} was exceeded!")
except Exception as exp:
cloudhunky_logger.exception(exp)
try:
logs = self.aci_client.container.list_logs(
self.resource_group.name, container_group_name, container.name)
self.aci_client.container_groups.delete(self.resource_group.name,
container_group_name)
except Exception as exp:
cloudhunky_logger.exception(exp)
return container_group_name, logs
memory_in_gb=c['properties']['resources']['requests']
['memoryInGB'])
unit = Container(
name=c['name'],
image=c['properties']['image'],
resources=ResourceRequirements(
requests=resources,
limits=None, # ResourceLimits()
),
environment_variables=config)
containers.append(unit)
registries = [
ImageRegistryCredential(**p)
for p in aci_config['properties']['imageRegistryCredentials']
]
group = ContainerGroup(
containers=containers,
os_type=aci_config['properties']['osType'],
location=aci_config['location'],
image_registry_credentials=registries,
restart_policy=aci_config['properties']['restartPolicy'])
logger.info('Creating container group')
try:
poller = aci_client.container_groups.create_or_update(
resource_group_name=rg,
