def _push_image(self, image, tag=None, auth=None, attempts=0):
""" Push an image to a remote Docker Registry.
Args:
image (str): The name of the Docker Image to be pushed.
tag (str): An optional tag for the Docker Image (default: 'latest')
auth: An optional Dict for the authentication.
attempts (int): The number of unsuccessful authentication attempts.
"""
if tag is None:
tag = 'latest'
if attempts == MAX_PUSH_ATTEMPTS:
err = 'Reached max attempts for pushing \
[{0}] with tag [{1}]'.format(image, tag)
logger.error(err)
raise FatalError(err)
logger.info('Pushing [{0}] with tag [{1}]'.format(image, tag))
result = self._client.images.push(image,
tag=tag,
auth_config=auth,
stream=True,
decode=True)
# Access Denied detection
for line in result:
# an error is occurred
if 'error' in line:
# access denied
if 'denied' or 'unauthorized' in line['error']:
logger.info('Access to the repository denied. \
Authentication failed.')
auth = create_auth_interactive()
self._push_image(image,
tag=tag,
auth=auth,
attempts=(attempts + 1))
else:
logger.error(
'Unknown error during push of [{0}]: {1}'.format(
image, line['error']))
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
logger.info('Image [{0}] with tag [{1}] successfully pushed.'.format(
image, tag))
def _image_authentication(self, src_image, src_tag=None, auth=None):
""" Handling Docker authentication if the image
is hosted on a private repository
Args:
src_image (str): The original image that needs
authentication for being fetched.
src_tag (str): The tag of the image above.
"""
if src_tag is None:
src_tag = 'latest'
logger.warning('[{0}:{1}] image may not exist or authentication \
is required'.format(src_image, src_tag))
res = ''
while res != 'YES':
res = (input('[{0}:{1}] is correct? [Yes] Continue \
[No] Abort\n'.format(src_image, src_tag))).upper()
if res == 'NO':
logger.error(
'Docker image [{0}:{1}] cannot be found, the operation \
is aborted by the user.\n(Hint: Check the TOSCA manifest.)'
.format(src_image, src_tag))
raise OperationAbortedByUser(
CommonErrorMessages._DEFAULT_OPERATION_ABORTING_ERROR_MSG)
attempts = 3
while attempts > 0:
logger.info('Authenticate with the Docker Repository..')
try:
if auth is None:
auth = create_auth_interactive(
user_text='Enter the username: '******'Enter the password: '******'UNAUTHORIZED' or 'NOT FOUND' in msg:
logger.info('Invalid username/password.')
else:
logger.exception(err)
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
attempts -= 1
logger.info('Authentication failed. \
You have [{}] more attempts.'.format(attempts))
# Check authentication failure
if attempts == 0:
logger.error(
'You have used all the authentication attempts. Abort.')
raise DockerAuthenticationFailedError(
'Authentication failed. Abort.')
def __init__(self, schema_path=None):
if schema_path is None:
schema_path = os.path.join(
os.path.dirname(__file__),
constants.DEFAULT_TOSKOSE_CONFIG_SCHEMA_PATH,
)
if not os.path.exists(schema_path):
raise FileNotFoundError('Toskose configuration schema not found.')
with open(schema_path, 'r') as f:
try:
self._config_schema = json.load(f)
except json.decoder.JSONDecodeError as err:
logger.error(err)
raise FatalError(
'The toskose configuration schema is corrupted. \
Validation cannot be done.')
except Exception as err:
logger.exception(err)
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
def search_runnable_commands(self, image, tag, pull=True):
""" Searching for ENTRYPOINT or CMD in a given image """
def _bind_shell(cmd_list):
""" e.g. ["/bin/sh", "-c", "echo", "hello!"] """
if cmd_list[0] in SUPPORTED_SHELLS:
return cmd_list
else:
# add a default shell on head
return (DEFAULT_SHELL + cmd_list)
full_name = '{0}:{1}'.format(image, tag)
logger.info('Analyzing the [{}] image'.format(full_name))
image = self._pull_image_with_auth(image, tag)
if image is not None:
output = self._client.api.inspect_image(full_name)
if not output or 'Config' not in output:
raise FatalError(
'Failed to inspect {} image'.format(full_name))
else:
entrypoint = output['Config']['Entrypoint']
cmd = output['Config']['Cmd']
commands = list()
if entrypoint and not cmd:
logger.debug(
'Detected only an ENTRYPOINT: {}'.format(entrypoint))
commands = _bind_shell(entrypoint)
elif not entrypoint and cmd:
logger.debug('Detected only a CMD: {}'.format(cmd))
commands = _bind_shell(cmd)
elif entrypoint and cmd:
logger.debug('Detected both ENTRYPOINT: \
{0} and CMD: {1}'.format(entrypoint, cmd))
# combining entrypoint (first) with cmd
commands = _bind_shell(entrypoint)
commands += cmd
else:
logger.info('the {} image is not runnable. \
Adding an infinite sleep'.format(full_name))
commands = DEFAULT_SHELL + ['sleep', 'infinity']
# add quotes
# [0]: /bin/bash [1]: -c
commands[2] = "\'" + commands[2]
commands[-1] = commands[-1] + "\'"
return ' '.join(commands)
def validate_config(self, config_path, tosca_model=None):
""" Validate a Toskose configuration file """
loader = Loader()
config = loader.load(config_path)
try:
jsonschema.validate(instance=config, schema=self._config_schema)
# if tosca_model is not None:
# ConfigValidator._validate_nodes(config, tosca_model)
except jsonschema.exceptions.ValidationError as err:
raise ValidationError(err.message)
except jsonschema.exceptions.SchemaError as err:
logger.error(err)
raise FatalError('The toskose configuration schema is corrupted. \
Validation cannot be done.')
def _generate_output_path(output_path=None):
""" Generate a default output path for toskose results. """
if output_path is None:
output_path = os.path.join(
os.getcwd(),
constants.DEFAULT_OUTPUT_PATH)
try:
if not os.path.exists(output_path):
os.makedirs(output_path)
except OSError as err:
logger.error('Failed to create {0} directory'.format(output_path))
logger.exception(err)
raise FatalError(CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
logger.info('Output dir {0} built'.format(output_path))
return output_path
def separate_full_image_name(full_name):
""" Separate a Docker image name in its attributes.
e.g. repository:port/user/name:tag ==>
{
'repository': 'repository:port',
'user': user,
'name': name,
'tag': tag
}
"""
def split_tag(name_tag):
if ':' in name_tag:
return tuple(splitted[2].split(':'))
else:
return name_tag, None
result = dict()
splitted = full_name.split('/')
if len(splitted) == 3:
# private repository
result['repository'] = splitted[0]
result['user'] = splitted[1]
result['name'], result['tag'] = split_tag(splitted[2])
elif len(splitted) == 2:
# Docker Hub
result['repository'] = None
result['user'] = splitted[0]
result['name'], result['tag'] = split_tag(splitted[1])
else:
raise FatalError(
'Cannot separate {} Docker image full name'.format(full_name))
return result
def validate_csar(csar_path):
""" Validate a TOSCA-based application compressed in a .CSAR archive. """
# TODO
# AGGIUNGI VALIDAZIONE CON SOMMELIER!!!
# E'ANCHE IN TESI!
# file existence
if not os.path.isfile(csar_path):
err_msg = 'Missing .CSAR archive file'
logger.error(err_msg)
raise FileNotFoundError(err_msg)
# file extension
if not csar_path.lower().endswith(_CSAR_ADMITTED_EXTENSIONS):
_, ext = os.path.splitext(csar_path)
err_msg = '{0} is an invalid file extension'.format(ext) \
if ext \
else 'file extension is not recognized'
logger.error(err_msg)
raise FileNotFoundError(err_msg)
# validate archive file
if not zipfile.is_zipfile(csar_path):
err_msg = '{0} is an invalid or corrupted archive'.format(csar_path)
logger.error(err_msg)
raise FileNotFoundError(err_msg)
logger.debug('Validating [{}]'.format(csar_path))
csar_metadata = {}
# validate csar structure
with zipfile.ZipFile(csar_path, 'r') as archive:
# TODO fix yaml error and remove it (workaround)
with suppress_stderr():
filelist = [e.filename for e in archive.filelist]
if _TOSCA_METADATA_PATH not in filelist:
logger.error('{0} does not contain a valid TOSCA.meta'.format(
csar_path))
raise MalformedCsarError(
CommonErrorMessages._DEFAULT_MALFORMED_CSAR_ERROR_MSG)
# validate TOSCA.meta
try:
# TODO !!!fix!!! YAMLLoadWarning: calling yaml.load()
# without Loader=... is deprecated, as the default Loader
# is unsafe.
# Please read https://msg.pyyaml.org/load for full details.
csar_metadata = yaml.load(archive.read(_TOSCA_METADATA_PATH))
if type(csar_metadata) is not dict:
logger.error('{0} is not a valid dictionary'.format(
_TOSCA_METADATA_PATH))
raise MalformedCsarError(
CommonErrorMessages._DEFAULT_MALFORMED_CSAR_ERROR_MSG)
except yaml.YAMLError as err:
logger.exception(err)
raise FatalError(CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)
# validate tosca metadata
for key in _TOSCA_METADATA_REQUIRED_KEYS:
if key not in csar_metadata:
logger.error('Missing {0} in {1}'.format(
key, _TOSCA_METADATA_PATH))
raise MalformedCsarError(
CommonErrorMessages._DEFAULT_MALFORMED_CSAR_ERROR_MSG)
# validate tosca manifest file
manifest = csar_metadata.get(_TOSCA_METADATA_MANIFEST_KEY)
if manifest is None or manifest not in filelist:
logger.error('{0} contains an invalid manifest reference \
or it does not exist'.format(_TOSCA_METADATA_PATH))
raise MalformedCsarError(
CommonErrorMessages._DEFAULT_MALFORMED_CSAR_ERROR_MSG)
# validate other tosca metadata
for option in _TOSCA_METADATA_OPTIONAL_KEYS:
if option not in csar_metadata:
logger.warning('Missing {0} option in {1}'.format(
option, _TOSCA_METADATA_PATH))
# validate tosca manifest (yaml)
with tempfile.TemporaryDirectory() as tmp_dir:
unpack_archive(csar_path, tmp_dir)
manifest_path = os.path.join(tmp_dir, manifest)
_validate_manifest(manifest_path)
return csar_metadata
def toskosed(self, csar_path,
config_path=None, output_path=None, enable_push=False):
"""
Entrypoint for the "toskoserization" process.
Args:
csar_path (str): The path to the TOSCA CSAR archive.
config_path (str): The path to the Toskose configuration file.
output_path (str): The path to the output directory.
enable_push (bool): Enable/Disable the auto-pushing of toskosed
images to Docker Registries.
Returns:
The docker-compose file representing the TOSCA-based application.
"""
if not os.path.exists(csar_path):
raise ValueError('The CSAR file {} doesn\'t exists'.format(
csar_path))
if config_path is not None:
if not os.path.exists(config_path):
raise ValueError(
'The configuration file {} doesn\'t exists'.format(
config_path))
if output_path is None:
logger.info('No output path detected. \
A default output path will be generated.')
output_path = Toskoserizator._generate_output_path()
if not os.path.exists(output_path):
raise ValueError('The output path {} doesn\'t exists'.format(
output_path))
csar_metadata = validate_csar(csar_path)
# temporary dir for unpacking data from .CSAR archive
# temporary dir for building docker images
with tempfile.TemporaryDirectory() as tmp_dir_context:
with tempfile.TemporaryDirectory() as tmp_dir_csar:
try:
unpack_archive(csar_path, tmp_dir_csar)
manifest_path = os.path.join(
tmp_dir_csar,
csar_metadata['Entry-Definitions'])
model = ToscaParser().build_model(manifest_path)
if config_path is None:
config_path = generate_default_config(model)
else:
ConfigValidator().validate_config(
config_path,
tosca_model=model)
# try to auto-complete config (if necessary)
config_path = generate_default_config(
model,
config_path=config_path)
toskose_model(model, config_path)
build_app_context(tmp_dir_context, model)
for container in model.containers:
if container.is_manager:
logger.info('Detected [{}] node [manager].'.format(
container.name))
template = ToskosingProcessType.TOSKOSE_MANAGER
elif container.hosted:
# if the container hosts sw components
# then it need to be toskosed
logger.info('Detected [{}] node.'.format(
container.name))
template = ToskosingProcessType.TOSKOSE_UNIT
else:
# the container doesn't host any sw component,
# left untouched
continue
ctx_path = os.path.join(
tmp_dir_context,
model.name,
container.name)
self._docker_manager.toskose_image(
src_image=container.image.name,
src_tag=container.image.tag,
dst_image=container.toskosed_image.name,
dst_tag=container.toskosed_image.tag,
context=ctx_path,
process_type=template,
app_name=model.name,
toskose_image=container.toskosed_image.base_name,
toskose_tag=container.toskosed_image.base_tag,
enable_push=enable_push
)
generate_compose(
tosca_model=model,
output_path=output_path,
)
self.quit()
except Exception as err:
logger.error(err)
raise FatalError(
CommonErrorMessages._DEFAULT_FATAL_ERROR_MSG)