def _build_push_ecr_image(tmp_folder: str, ecr_image: str, platform: str,
auth_token: Set) -> str:
try:
dclient = docker.from_env()
except docker.errors.DockerException:
raise Exception(
"Error getting docker client. Check if current user has the correct permissions (docker group)."
)
logger.info('Building new ECR image: %s' % ecr_image)
dclient.images.build(path=tmp_folder,
tag=ecr_image,
pull=True,
platform=platform)
# Login to the ECR registry
# Known issue it does not work in Widnows WSL environment
registry = os.path.dirname(ecr_image)
logger.info('Login to ECR registry %s' % registry)
dclient.login(username=auth_token[0],
password=auth_token[1],
registry=registry)
# Push the image, and change it in the container image to use it insteads of the user one
logger.info('Pushing new image to ECR ...')
for line in dclient.images.push(ecr_image, stream=True, decode=True):
logger.debug(line)
if 'error' in line:
raise Exception("Error pushing image: %s" %
line['errorDetail']['message'])
return "%s:latest" % ecr_image
def execute_command_with_msg(command: List[str], cmd_wd: Optional[str]=None,
cli_msg: str='') -> str:
"""Execute the specified command and return the result."""
cmd_out = subprocess.check_output(command, cwd=cmd_wd).decode('utf-8')
logger.debug(cmd_out)
logger.info(cli_msg)
return cmd_out[:-1]
def create_service(self, **kwargs: Dict) -> Dict:
"""Creates a new OSCAR service."""
logger.debug('Creating OSCAR service.')
res = requests.post(f'{self.endpoint}{self._SERVICES_PATH}',
auth=(self.auth_user, self.auth_password),
verify=self.ssl_verify,
json=kwargs)
# Raise a ServiceCreationError if the return code is not 201
if res.status_code != 201:
raise excp.ServiceCreationError(service_name=kwargs['name'],
error_msg=_get_error_msg(res))
def delete_service(self, service_name: str) -> None:
"""Deletes an OSCAR service."""
logger.debug('Deleting OSCAR service.')
res = requests.delete(
f'{self.endpoint}{self._SERVICES_PATH}/{service_name}',
auth=(self.auth_user, self.auth_password),
verify=self.ssl_verify)
# Raise a ServiceDeletionError if the return code is not 204
if res.status_code != 204:
raise excp.ServiceDeletionError(service_name=service_name,
error_msg=_get_error_msg(res))
def list_functions(self, next_token: Optional[str] = None) -> List:
"""Returns a list of your Lambda functions."""
logger.debug("Listing lambda functions.")
functions = []
kwargs = {}
if next_token:
kwargs['Marker'] = next_token
functions_info = self.client.list_functions(**kwargs)
if 'Functions' in functions_info and functions_info['Functions']:
functions.extend(functions_info['Functions'])
if 'NextMarker' in functions:
functions.extend(
self.list_layers(next_token=functions_info['NextMarker']))
return functions
def list_layers(self, next_token: Optional[str] = None) -> List:
"""Lists function layers and shows information
about the latest version of each."""
logger.debug("Listing lambda layers.")
layers = []
kwargs = {}
if next_token:
kwargs['Marker'] = next_token
layers_info = self.client.list_layers(**kwargs)
if 'Layers' in layers_info and layers_info['Layers']:
layers.extend(layers_info['Layers'])
if 'NextMarker' in layers_info:
layers.extend(
self.list_layers(next_token=layers_info['NextMarker']))
return layers
def list_layer_versions(self,
layer_name: str,
next_token: Optional[str] = None) -> str:
"""Lists the versions of an AWS Lambda layer."""
logger.debug(f'Listing versions of lambda layer "{layer_name}".')
versions = []
kwargs = {'LayerName': layer_name}
if next_token:
kwargs['Marker'] = next_token
layer_versions_info = self.client.list_layer_versions(**kwargs)
if 'LayerVersions' in layer_versions_info and layer_versions_info[
'LayerVersions']:
versions.extend(layer_versions_info['LayerVersions'])
if 'NextMarker' in layer_versions_info:
versions.extend(
self.list_layer_versions(
layer_name, next_token=layer_versions_info['NextMarker']))
return versions
def get_supervisor_zip(resources_info: Dict,
supervisor_version: str) -> str:
"""Get from cache or download supervisor zip."""
asset_name = ContainerImage.get_asset_name(
resources_info.get('lambda'))
cached, supervisor_zip_path = SupervisorUtils.is_supervisor_asset_cached(
asset_name, supervisor_version)
if cached:
# It is cached, do not download again
logger.debug(
'Using supervisor asset cached file: ver: %s, asset: %s' %
(supervisor_version, asset_name))
return supervisor_zip_path
else:
logger.debug(
'Downloading supervisor asset file: ver: %s, asset: %s' %
(supervisor_version, asset_name))
return SupervisorUtils.download_supervisor_asset(
supervisor_version, asset_name, supervisor_zip_path)
def delete_compute_environment(self, **kwargs: Dict) -> Dict:
"""Deletes a compute environment."""
logger.debug("Deleting compute environment.")
return self.client.delete_compute_environment(**kwargs)
def create_job_queue(self, **kwargs: Dict) -> Dict:
"""Creates a new job queue."""
logger.debug("Creating job queue.")
return self.client.create_job_queue(**kwargs)
def create_function(self, **kwargs: Dict) -> Dict:
"""Creates a new Lambda function."""
logger.debug("Creating lambda function.")
return self.client.create_function(**kwargs)
def publish_layer_version(self, **kwargs: Dict) -> Dict:
"""Creates a function layer from a ZIP archive."""
logger.debug("Publishing lambda layer.")
return self.client.publish_layer_version(**kwargs)
def register_job_definition(self, **kwargs: Dict) -> Dict:
"""Registers a new job definition."""
logger.debug("Registering job definition.")
return self.client.register_job_definition(**kwargs)
def describe_compute_environments(self, **kwargs: Dict) -> Dict:
"""Describes compute environments."""
logger.debug("Describing Compute Environment.")
return self.client.describe_compute_environments(**kwargs)
def describe_job_queues(self, **kwargs: Dict) -> Dict:
"""Describe a new job queue."""
logger.debug("Describing job queue.")
return self.client.describe_job_queues(**kwargs)
def describe_jobs(self, **kwargs: Dict) -> Dict:
"""Describes a batch job."""
logger.debug("Describing a job.")
return self.client.describe_jobs(**kwargs)
def describe_job_definitions(self, **kwargs: Dict) -> Dict:
"""Describes a list of job definitions."""
logger.debug("Describing job definition.")
return self.client.describe_job_definitions(**kwargs)
def create_compute_environment(self, **kwargs: Dict) -> Dict:
"""Creates a new compute environment."""
logger.debug("Creating compute environment.")
return self.client.create_compute_environment(**kwargs)
def update_compute_environment(self, **kwargs: Dict) -> Dict:
"""Updates a compute environment."""
logger.debug("Updating compute environment.")
return self.client.update_compute_environment(**kwargs)
def delete_job_queue(self, **kwargs: Dict) -> Dict:
"""Deletes a job queue."""
logger.debug("Deleting job queue.")
return self.client.delete_job_queue(**kwargs)
def update_job_queue(self, **kwargs: Dict) -> Dict:
"""Updates a job queue."""
logger.debug("Updating job queue.")
return self.client.update_job_queue(**kwargs)
def deregister_job_definition(self, **kwargs: Dict) -> Dict:
"""Deregisters an AWS Batch job definition."""
logger.debug("Deleting job definition.")
return self.client.deregister_job_definition(**kwargs)
