def _get_artifact_paths_for_path(self, path):
"""
Get artifact paths for a local filesystem path. We support path to
an artifact file or a directory containing artifact files as its
immediate children, i.e., we do not deal with nested artifact
directories at this moment.
If a file or directory is not found, raise an exception.
Args:
path (str): Local path
Returns:
list: A list of artifact paths
"""
artifact_paths = []
if os.path.isfile(path):
artifact_paths.append(path)
elif os.path.isdir(path):
if os.listdir(path) == []:
raise NuleculeException("Artifact directory %s is empty" %
path)
for dir_child in os.listdir(path):
dir_child_path = os.path.join(path, dir_child)
if dir_child.startswith('.') or os.path.isdir(dir_child_path):
continue
artifact_paths.append(dir_child_path)
else:
raise NuleculeException("Unable to find artifact %s" % path)
return artifact_paths
def render(self, provider_key=None, dryrun=False):
"""
Render the artifact files for the Nuelcule component. If the component
is an external Nulecule application, recurse into it to load it and
render it's artifacts. If provider_key is specified, render artifacts
only for that provider, else, render artifacts for all providers.
Args:
provider_key (str or None): Provider name.
Returns:
None
"""
if self._app:
self._app.render(provider_key=provider_key, dryrun=dryrun)
return
if self.artifacts is None:
raise NuleculeException(
"No artifacts specified in the Nulecule file")
if provider_key and provider_key not in self.artifacts:
raise NuleculeException(
"Data for provider \"%s\" are not part of this app" %
provider_key)
context = self.get_context()
for provider in self.artifacts:
if provider_key and provider != provider_key:
continue
for artifact_path in self.get_artifact_paths_for_provider(
provider):
self.rendered_artifacts[provider].append(
self.render_artifact(artifact_path, context, provider))
def __init__(self, app_spec, destination=None, answers_file=None):
"""
init function for NuleculeManager. Sets a few instance variables.
Args:
app_spec: either a path to an unpacked nulecule app or a
container image name where a nulecule can be found
destination: where to unpack a nulecule to if it isn't local
"""
self.answers = copy.deepcopy(DEFAULT_ANSWERS)
self.answers_format = None
self.answers_file = None # The path to an answer file
self.app_path = None # The path where the app resides or will reside
self.image = None # The container image to pull the app from
# Adjust app_spec, destination, and answer file paths if absolute.
if os.path.isabs(app_spec):
app_spec = os.path.join(Utils.getRoot(), app_spec.lstrip('/'))
if destination and os.path.isabs(destination):
destination = os.path.join(Utils.getRoot(),
destination.lstrip('/'))
if answers_file and os.path.isabs(answers_file):
answers_file = os.path.join(Utils.getRoot(),
answers_file.lstrip('/'))
# Determine if the user passed us an image or a path to an app
if not os.path.exists(app_spec):
self.image = app_spec
else:
self.app_path = app_spec
# Doesn't make sense to provide an app path and destination
if self.app_path and destination:
raise NuleculeException(
"You can't provide a local path and destination.")
# If the user provided an image, make sure we have a destination
if self.image:
if destination:
self.app_path = destination
else:
self.app_path = Utils.getNewAppCacheDir(self.image)
logger.debug("NuleculeManager init app_path: %s", self.app_path)
logger.debug("NuleculeManager init image: %s", self.image)
# Set where the main nulecule file should be
self.main_file = os.path.join(self.app_path, MAIN_FILE)
# If user provided a path to answers then make sure it exists. If they
# didn't provide one then use the one in the app dir if it exists.
if answers_file:
self.answers_file = answers_file
if not os.path.isfile(self.answers_file):
raise NuleculeException("Path for answers doesn't exist: %s" %
self.answers_file)
else:
if os.path.isfile(os.path.join(self.app_path, ANSWERS_FILE)):
self.answers_file = os.path.join(self.app_path, ANSWERS_FILE)
def load_from_path(cls,
src,
config=None,
namespace=GLOBAL_CONF,
nodeps=False,
dryrun=False,
update=False):
"""
Load a Nulecule application from a path in the source path itself, or
in the specified destination path.
Args:
src (str): Path to load Nulecule application from.
config (dict): Config data for Nulecule application.
namespace (str): Namespace for Nulecule application.
nodeps (bool): Do not pull external applications if True.
dryrun (bool): Do not make any change to underlying host.
update (bool): Update existing application if True, else reuse it.
Returns:
A Nulecule instance or None in case of some dry run (fetching
an image).
"""
nulecule_path = os.path.join(src, MAIN_FILE)
if os.path.exists(nulecule_path):
with open(nulecule_path, 'r') as f:
nulecule_data = f.read()
else:
raise NuleculeException(
"No Nulecule file exists in directory: %s" % src)
if dryrun and not os.path.exists(nulecule_path):
raise NuleculeException(
"Fetched Nulecule components are required to initiate dry-run. "
"Please specify your app via atomicapp --dry-run /path/to/your-app"
)
# By default, AnyMarkup converts all formats to YAML when parsing.
# Thus the rescue works either on JSON or YAML.
try:
nulecule_data = anymarkup.parse(nulecule_data)
except (yaml.parser.ParserError, AnyMarkupError), e:
line = re.search('line (\d+)', str(e)).group(1)
column = re.search('column (\d+)', str(e)).group(1)
output = ""
for i, l in enumerate(nulecule_data.splitlines()):
if (i == int(line) -
1) or (i == int(line)) or (i == int(line) + 1):
output += "%s %s\n" % (str(i), str(l))
raise NuleculeException(
"Failure parsing %s file. Validation error on line %s, column %s:\n%s"
% (nulecule_path, line, column, output))
def load_from_path(cls,
src,
config=None,
namespace=GLOBAL_CONF,
nodeps=False,
dryrun=False,
update=False):
"""
Load a Nulecule application from a path in the source path itself, or
in the specified destination path.
Args:
src (str): Path to load Nulecule application from.
config (dict): Config data for Nulecule application.
namespace (str): Namespace for Nulecule application.
nodeps (bool): Do not pull external applications if True.
dryrun (bool): Do not make any change to underlying host.
update (bool): Update existing application if True, else reuse it.
Returns:
A Nulecule instance or None in case of some dry run (installing
from image).
"""
nulecule_path = os.path.join(src, MAIN_FILE)
if dryrun and not os.path.exists(nulecule_path):
raise NuleculeException(
"Installed Nulecule components are required to initiate dry-run"
)
nulecule_data = anymarkup.parse_file(nulecule_path)
nulecule = Nulecule(config=config,
basepath=src,
namespace=namespace,
**nulecule_data)
nulecule.load_components(nodeps, dryrun)
return nulecule
def genanswers(self, dryrun=False, answers_format=None, **kwargs):
"""
Renders artifacts and then generates an answer file. Finally
copies answer file to the current working directory.
Args:
dryrun (bool): Do not make any change to the host system if True
answers_format (str): File format for writing sample answers file
kwargs (dict): Extra keyword arguments
Returns:
None
"""
self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT
# Check to make sure an answers.conf file doesn't exist already
answers_file = os.path.join(os.getcwd(), ANSWERS_FILE)
if os.path.exists(answers_file):
raise NuleculeException(
"Can't generate answers.conf over existing file")
# Call unpack to get the app code
self.nulecule = self.unpack(update=False,
dryrun=dryrun,
config=self.answers)
self.nulecule.load_config(config=self.nulecule.config,
skip_asking=True)
# Get answers and write them out to answers.conf in cwd
answers = self._get_runtime_answers(self.nulecule.config, None)
self._write_answers(answers_file, answers, answers_format)
def apply_pointers(self, content, params):
"""
Let's apply all the json pointers!
Valid params in Nulecule:
param1:
- /spec/containers/0/ports/0/hostPort
- /spec/containers/0/ports/0/hostPort2
or
param1:
- /spec/containers/0/ports/0/hostPort, /spec/containers/0/ports/0/hostPort2
Args:
content (str): content of artifact file
params (dict): list of params with pointers to replace in content
Returns:
str: content with replaced pointers
Todo:
In the future we need to change this to detect haml, yaml, etc as we add more providers
Blocked by: github.com/bkabrda/anymarkup-core/blob/master/anymarkup_core/__init__.py#L393
"""
obj = anymarkup.parse(content)
if type(obj) != dict:
logger.debug(
"Artifact file not json/haml, assuming it's $VARIABLE substitution"
)
return content
if params is None:
# Nothing to do here!
return content
for name, pointers in params.items():
if not pointers:
logger.warning("Could not find pointer for %s" % name)
continue
for pointer in pointers:
try:
resolve_pointer(obj, pointer)
set_pointer(obj, pointer, name)
logger.debug("Replaced %s pointer with %s param" %
(pointer, name))
except JsonPointerException:
logger.debug("Error replacing %s with %s" %
(pointer, name))
logger.debug("Artifact content: %s", obj)
raise NuleculeException(
"Error replacing pointer %s with %s." %
(pointer, name))
return anymarkup.serialize(obj, format="json")
def extract_nulecule_data(self, image, source, dest, update=False):
"""
Extract the Nulecule contents from a container into a destination
directory.
Args:
image (str): Docker image name
source (str): Source directory in Docker image to copy from
dest (str): Path to destination directory on host
update (bool): Update destination directory if it exists when
True
Returns:
None
"""
logger.info(
'Extracting Nulecule data from image %s to %s' % (image, dest))
if self.dryrun:
return
# Create a temporary directory for extraction
tmpdir = '/tmp/nulecule-{}'.format(uuid.uuid1())
self.extract_files(image, source=source, dest=tmpdir)
# If the application already exists locally then need to
# make sure the local app id is the same as the one requested
# on the command line.
mainfile = os.path.join(dest, MAIN_FILE)
tmpmainfile = os.path.join(tmpdir, MAIN_FILE)
if os.path.exists(mainfile):
existing_id = Utils.getAppId(mainfile)
new_id = Utils.getAppId(tmpmainfile)
cockpit_logger.info("Loading app_id %s" % new_id)
if existing_id != new_id:
raise NuleculeException(
"Existing app (%s) and requested app (%s) differ" %
(existing_id, new_id))
# If app exists and no update requested then move on
if update:
logger.info("App exists locally. Performing update...")
else:
logger.info("App exists locally and no update requested")
return
# Copy files from tmpdir into place
logger.debug('Copying nulecule data from %s to %s' % (tmpdir, dest))
Utils.copy_dir(tmpdir, dest, update)
# Clean up tmpdir
logger.debug('Removing tmp dir: %s' % tmpdir)
Utils.rm_dir(tmpdir)
# Set the proper permissions on the extracted folder
Utils.setFileOwnerGroup(dest)
def _process_answers(self):
"""
Processes answer files to load data from them and then merges
any cli provided answers into the config.
NOTE: This function should be called once on startup and then
once more after the application has been extracted, but only
if answers file wasn't found on the first invocation. The idea
is to allow for people to embed an answers file in the application
if they want, which won't be available until after extraction.
Returns:
None
"""
app_path_answers = os.path.join(self.app_path, ANSWERS_FILE)
# If the user didn't provide an answers file then check the app
# dir to see if one exists.
if not self.answers_file:
if os.path.isfile(app_path_answers):
self.answers_file = app_path_answers
# At this point if we have an answers file, load it
if self.answers_file:
# If this is a url then download answers file to app directory
if urlparse.urlparse(self.answers_file).scheme != "":
logger.debug("Retrieving answers file from: {}".format(
self.answers_file))
with open(app_path_answers, 'w+') as f:
stream = urllib.urlopen(self.answers_file)
f.write(stream.read())
self.answers_file = app_path_answers
# Check to make sure the file exists
if not os.path.isfile(self.answers_file):
raise NuleculeException(
"Provided answers file doesn't exist: {}".format(
self.answers_file))
# Load answers
self.answers = Utils.loadAnswers(self.answers_file)
# If there is answers data from the cli then merge it in now
if self.cli_answers:
for k, v in self.cli_answers.iteritems():
self.answers[GLOBAL_CONF][k] = v
def get_artifact_paths_for_provider(self, provider_key):
"""
Get artifact file paths of a Nulecule component for a provider.
Args:
provider_key (str): Provider name
Returns:
list: A list of artifact paths.
"""
artifact_paths = []
artifacts = self.artifacts.get(provider_key)
# If there are no artifacts for the requested provider then error
# This can happen for incorrectly named inherited provider (#435)
if artifacts is None:
raise NuleculeException(
"No artifacts for provider {}".format(provider_key))
for artifact in artifacts:
# Convert dict if the Nulecule file references "resource"
if isinstance(artifact, dict) and artifact.get(RESOURCE_KEY):
artifact = artifact[RESOURCE_KEY]
logger.debug("Resource xpath added: %s" % artifact)
# Sanitize the file structure
if isinstance(artifact, basestring):
path = Utils.sanitizePath(artifact)
path = os.path.join(self.basepath, path) \
if path[0] != '/' else path
artifact_paths.extend(self._get_artifact_paths_for_path(path))
# Inherit if inherit name is referenced
elif isinstance(artifact, dict) and artifact.get(INHERIT_KEY) and \
isinstance(artifact.get(INHERIT_KEY), list):
for inherited_provider_key in artifact.get(INHERIT_KEY):
artifact_paths.extend(
self.get_artifact_paths_for_provider(
inherited_provider_key)
)
else:
logger.error('Invalid artifact file')
return artifact_paths
def get_provider(self, provider_key=None, dry=False):
"""
Get provider key and provider instance.
Args:
provider_key (str or None): Name of provider
dry (bool): Do not make change to the host system while True
Returns:
tuple: (provider key, provider instance)
"""
# If provider_key isn't provided via CLI, let's grab it the configuration
if provider_key is None:
provider_key = self.config.get(GLOBAL_CONF)[PROVIDER_KEY]
provider_class = self.plugin.getProvider(provider_key)
if provider_class is None:
raise NuleculeException("Invalid Provider - '{}', provided in "
"answers.conf (choose from {})".format(
provider_key, ', '.join(PROVIDERS)))
return provider_key, provider_class(self.get_context(), self.basepath,
dry)
def extract(self, image, source, dest, update=False):
"""
Extracts content from a directory in a Docker image to specified
destination.
Args:
image (str): Docker image name
source (str): Source directory in Docker image to copy from
dest (str): Path to destination directory on host
update (bool): Update destination directory if it exists when
True
Returns:
None
"""
logger.info('Extracting nulecule data from image: %s to %s' %
(image, dest))
if self.dryrun:
return
# Create dummy container
run_cmd = [
self.docker_cli, 'create', '--entrypoint', '/bin/true', image
]
logger.debug('Creating docker container: %s' % ' '.join(run_cmd))
container_id = subprocess.check_output(run_cmd).strip()
# Copy files out of dummy container to tmpdir
tmpdir = '/tmp/nulecule-{}'.format(uuid.uuid1())
cp_cmd = [
self.docker_cli, 'cp',
'%s:/%s' % (container_id, source), tmpdir
]
logger.debug('Copying data from Docker container: %s' %
' '.join(cp_cmd))
subprocess.call(cp_cmd)
# There has been some inconsistent behavior where docker cp
# will either copy out the entire dir /APP_ENT_PATH/*files* or
# it will copy out just /*files* without APP_ENT_PATH. Detect
# that here and adjust accordingly.
src = os.path.join(tmpdir, APP_ENT_PATH)
if not os.path.exists(src):
src = tmpdir
# If the application already exists locally then need to
# make sure the local app id is the same as the one requested
# on the command line.
mainfile = os.path.join(dest, MAIN_FILE)
tmpmainfile = os.path.join(src, MAIN_FILE)
if os.path.exists(mainfile):
existing_id = Utils.getAppId(mainfile)
new_id = Utils.getAppId(tmpmainfile)
cockpit_logger.info("Loading app_id %s ." % new_id)
if existing_id != new_id:
raise NuleculeException(
"Existing app (%s) and requested app (%s) differ" %
(existing_id, new_id))
# If app exists and no update requested then move on
if update:
logger.info("App exists locally. Performing update...")
else:
logger.info("App exists locally and no update requested")
return
# Copy files
logger.debug('Copying nulecule data from %s to %s' % (src, dest))
Utils.copy_dir(src, dest, update)
logger.debug('Removing tmp dir: %s' % tmpdir)
Utils.rm_dir(tmpdir)
# Clean up dummy container
rm_cmd = [self.docker_cli, 'rm', '-f', container_id]
logger.debug('Removing Docker container: %s' % ' '.join(rm_cmd))
subprocess.call(rm_cmd)