def _copy_impl(self, target):
if not os.path.exists(self.path):
cache = tools.cfg.get('common', {}).get('cache_url', None)
# If cache_url is specified in Concreate configuration
# file - try to fetch the 'path' artifact from cacher
# even if it was not defined as 'url'.
if cache:
try:
self._download_file(self.path, target)
return target
except Exception as ex:
logger.exception(ex)
raise ConcreateError(
"Could not download resource '%s' from cache" %
self.name)
else:
raise ConcreateError(
"Could not copy resource '%s', source path does not exist. Make sure you provided correct path"
% self.name)
logger.debug("Copying repository from '%s' to '%s'." %
(self.path, target))
if os.path.isdir(self.path):
shutil.copytree(self.path, target)
else:
shutil.copy(self.path, target)
return target
def copy_module_to_target(name, version, target):
"""Copies a module from args.target/repo/... directory into
args.target/image/modules/... and update module path to be
the new location
Arguments:
name - name of the module to lookup in modules list
version - version of the module to used
target - directory where module will be copied
Returns instance of copied module.
"""
if not os.path.exists(target):
os.makedirs(target)
# FIXME: version checking
candidates = [m for m in modules if name == m.name]
if not candidates:
raise ConcreateError("Cannot find requested module: '%s'" % name)
for module in candidates:
if not version or version == module.get('version', None):
dest = os.path.join(target, module.name)
# if module is already copied, check that the version is correct
if os.path.exists(dest) and version:
check_module_version(dest, version)
if not os.path.exists(dest):
logger.debug("Copying module '%s' to: '%s'" % (name, dest))
shutil.copytree(module.path, dest)
return module
raise ConcreateError("Cannot find requested module: '%s', version:'%s'." %
(name, version))
def check_prerequisities(self):
try:
subprocess.check_output(['docker', 'info'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
raise ConcreateError("Docker build engine needs docker installed and configured, error: %s"
% ex.output)
except Exception as ex:
raise ConcreateError("Docker build engine needs docker installed and configured!", ex)
def __init__(self, target):
"""Check if behave and docker is installed properly"""
self.target = os.path.abspath(target)
try:
subprocess.check_output(['behave', '--version'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
raise ConcreateError("Test Runner needs 'behave' installed, '%s'" %
ex.output)
except Exception as ex:
raise ConcreateError(
"Test Runner needs behave installed!", ex)
def check_prerequisities(self):
try:
subprocess.check_output(['rhpkg', 'help'],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
raise ConcreateError(
"OSBS build engine needs 'rhpkg' tools installed, error: %s" %
ex.output)
except Exception as ex:
raise ConcreateError(
"OSBS build engine needs 'rhpkg' tools installed!", ex)
def build(self, build_args):
"""After the source siles are generated, the container image can be built.
We're using Docker to build the image currently.
This can be changed by specifying the tags in CLI using --build-tags option.
Args:
build_tags - a list of image tags
"""
tags = build_args.tags
cmd = ["docker", "build"]
# Custom tags for the container image
logger.debug("Building image with tags: '%s'" % "', '".join(tags))
for tag in tags:
cmd.extend(["-t", tag])
logger.info("Building container image...")
cmd.append(os.path.join(self.target, 'image'))
logger.debug("Running Docker build: '%s'" % " ".join(cmd))
try:
subprocess.check_call(cmd)
logger.info("Image built and available under following tags: %s" %
", ".join(tags))
except:
raise ConcreateError("Image build failed, see logs above.")
def merge_lists(list1, list2):
""" Merges two lists handling embedded dictionaries via 'name' as a key
In a case of simple type values are appended.
Args:
list1, list2 - list to merge
Returns merged list
"""
list1_dicts = [x for x in list1 if isinstance(x, dict)]
for v2 in list2:
if isinstance(v2, dict):
if 'name' not in v2:
raise KeyError("The 'name' key was not found in dict: %s" % v2)
if v2['name'] not in [x['name'] for x in list1_dicts]:
list1.append(v2)
else:
for v1 in list1_dicts:
if v2['name'] == v1['name']:
merge_dictionaries(v1, v2)
elif isinstance(v2, list):
raise ConcreateError("Cannot merge list of lists")
else:
if v2 not in list1:
list1.append(v2)
return list1
def check_module_version(path, version):
descriptor = Module(
tools.load_descriptor(os.path.join(path, 'module.yaml')), path)
if descriptor.version != version:
raise ConcreateError(
"Requested conflicting version '%s' of module '%s'" %
(version, descriptor['name']))
def copy_module_to_target(name, version, target):
"""Copies a module from args.target/repo/... directory into
args.target/image/modules/... and update module path to be
the new location
Arguments:
name - name of the module to lookup in modules list
version - version of the module to used
target - directory where module will be copied
Returns instance of copied module.
"""
if not os.path.exists(target):
os.makedirs(target)
# FIXME: version checking
for module in modules:
if name == module.name:
dest = os.path.join(target, name)
logger.info('Preparing module %s' % module.name)
if os.path.exists(dest):
# FIXME check version
return module
shutil.copytree(module.path, dest)
module.path = dest
return module
raise ConcreateError("Cannot find requested module: '%s'" % name)
def run(self, image, run_tags):
"""Run test suite"""
cmd = ['behave',
'--junit',
'--junit-directory', 'results',
'-t', '~ignore',
'--no-skipped',
'-D', 'IMAGE=%s' % image]
for tag in run_tags:
if ':' in tag:
test_tag = tag.split(':')[0]
cmd.append('-t')
if '/' in tag:
cmd.append("@%s,@%s" % (test_tag.split('/')[0], test_tag))
else:
cmd.append(tag)
# Check if we're running runtests on CI or locally
# If we run tests locally - skip all features that
# are marked with the @ci annotation
if getpass.getuser() != "jenkins":
cmd.append("-t")
cmd.append("~ci ")
logger.debug("Running '%s'" % ' '.join(cmd))
try:
subprocess.check_call(cmd, stderr=subprocess.STDOUT, cwd=os.path.join(self.target, 'test'))
except:
raise ConcreateError("Test execution failed, please consult output above")
def copy(self, target_dir=os.getcwd()):
target = os.path.join(target_dir, self.target_file_name())
logger.debug("Fetching resource '%s'" % (self.name))
overwrite = False
if os.path.exists(target):
try:
self.__verify(target)
except:
logger.debug("Local resource verification failed")
overwrite = True
if os.path.exists(target) and not overwrite:
logger.debug("Local resource '%s' exists and is valid, skipping" %
self.name)
return target
try:
self._copy_impl(target)
except Exception as ex:
logger.warn(
"Concreate is not able to fetch resource '%s' automatically. You can manually place required artifact as '%s'"
% (self.name, target))
if self.description:
logger.info(self.description)
# exception is fatal we be logged before Concreate dies
raise ConcreateError(
"Error copying resource: '%s'. See logs for more info." %
self.name, ex)
self.__verify(target)
return target
def prepare(self, descriptor):
"""Prepares dist-git repository for OSBS build."""
repository_key = descriptor.get('osbs', {}).get('repository', {})
repository = repository_key.get('name')
branch = repository_key.get('branch')
if not (repository and branch):
raise ConcreateError(
"OSBS builder needs repostiory and branch provided!")
self.dist_git_dir = os.path.join(os.path.expanduser('~/.concreate.d'),
'osbs', repository)
if not os.path.exists(os.path.dirname(self.dist_git_dir)):
os.makedirs(os.path.dirname(self.dist_git_dir))
self.dist_git = Git(self.dist_git_dir, self.target, repository, branch)
self.dist_git.prepare()
self.dist_git.clean()
self.update_osbs_image_source()
artifacts = [a['name'] for a in descriptor.get('artifacts', [])]
self.update_lookaside_cache(artifacts)
def load_descriptor(descriptor_path, schema_type):
""" parses descriptor and validate it against requested schema type
Args:
schema_type - type of schema (module/image)
descriptor_path - path to image/modules descriptor to be validated
Returns validated schema
"""
logger.debug("Loading %s descriptor from path '%s'."
% (schema_type,
descriptor_path))
if not os.path.exists(descriptor_path):
raise ConcreateError('Cannot find provided descriptor file')
common_schema_path = os.path.join(os.path.dirname(__file__),
'schema',
'common.yaml')
schema = {}
# Read main schema definition
with open(common_schema_path, 'r') as fh:
schema = yaml.safe_load(fh)
specific_schema_path = os.path.join(os.path.dirname(__file__),
'schema',
'%s.yaml' % schema_type)
# Read schema definition for specific type
with open(specific_schema_path, 'r') as fh:
specific_schema = yaml.safe_load(fh)
schema = merge_dictionaries(schema, specific_schema, True)
descriptor = {}
with open(descriptor_path, 'r') as fh:
descriptor = yaml.safe_load(fh)
core = Core(source_data=descriptor,
schema_data=schema, allow_assertions=True)
try:
return core.validate(raise_exception=True)
except Exception as ex:
raise ConcreateError("Cannot validate schema: %s" % (descriptor_path),
ex)
def check_schema_version(self):
""" Check supported schema version """
if self.descriptor['schema_version'] != schema_version:
raise ConcreateError(
"Schema version: '%s' is not supported by current version."
" This version supports schema version: '%s' only."
" To build this image please install concreate version: '%s'" %
(self.descriptor['schema_version'], schema_version,
self.descriptor['schema_version']))
def merge(self, descriptor):
""" Merges two descriptors in a way, that arrays are appended
and duplicit values are kept
Args:
descriptor - a concreate descritor
"""
try:
self.descriptor = concreate.tools.merge_dictionaries(self.descriptor, descriptor)
except KeyError as ex:
logger.debug(ex, exc_info=True)
raise ConcreateError("Cannot merge descriptors, see log message for more information")
def __validate(self):
for schema in self.schemas:
core = Core(source_data=self.descriptor,
schema_data=schema, allow_assertions=True)
try:
core.validate(raise_exception=True)
return
except Exception as ex:
# We log this as debug, because we support multiple schemas
logger.debug("Schema validation failed: %s" % ex)
raise ConcreateError("Cannot validate schema: %s" % (self.__class__.__name__))
def update_lookaside_cache(self):
logger.info("Updating lookaside cache...")
if not self.artifacts:
return
cmd = ["rhpkg", "new-sources"] + self.artifacts
logger.debug("Executing '%s'" % cmd)
with Chdir(self.dist_git_dir):
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
logger.error("Cannot run '%s', ouput: '%s'" % (cmd, ex.output))
raise ConcreateError("Cannot update sources.")
logger.info("Update finished.")
def _download_file(self, url, destination, use_cache=True):
""" Downloads a file from url and save it as destination """
if use_cache:
url = self.__substitute_cache_url(url)
logger.debug("Downloading from '%s' as %s" % (url, destination))
parsedUrl = urlparse(url)
if parsedUrl.scheme == 'file' or not parsedUrl.scheme:
if os.path.isdir(parsedUrl.path):
shutil.copytree(parsedUrl.path, destination)
else:
shutil.copy(parsedUrl.path, destination)
elif parsedUrl.scheme in ['http', 'https']:
verify = tools.cfg.get('common', {}).get('ssl_verify', True)
if str(verify).lower() == 'false':
verify = False
ctx = ssl.create_default_context()
if not verify:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
res = urlopen(url, context=ctx)
if res.getcode() != 200:
raise ConcreateError("Could not download file from %s" % url)
with open(destination, 'wb') as f:
while True:
chunk = res.read(1024)
if not chunk:
break
f.write(chunk)
else:
raise ConcreateError("Unsupported URL scheme: %s" % (url))
def load_descriptor(descriptor_path):
""" parses descriptor and validate it against requested schema type
Args:
descriptor_path - path to image/modules descriptor to be validated
Returns descriptor as a dictionary
"""
logger.debug("Loading descriptor from path '%s'." % descriptor_path)
if not os.path.exists(descriptor_path):
raise ConcreateError('Cannot find provided descriptor file')
with open(descriptor_path, 'r') as fh:
return yaml.safe_load(fh)
def __new__(cls, build_engine, target, params={}):
if cls is Builder:
if 'docker' == build_engine:
# import is delayed until here to prevent circular import error
from concreate.builders.docker import DockerBuilder as BuilderImpl
logger.info("Using Docker builder to build the image.")
elif 'osbs' == build_engine:
# import is delayed until here to prevent circular import error
from concreate.builders.osbs import OSBSBuilder as BuilderImpl
logger.info("Using OSBS builder to build the image.")
else:
raise ConcreateError("Builder engine %s is not supported" %
build_engine)
return super(Builder, cls).__new__(BuilderImpl)
def load_descriptor(descriptor):
""" parses descriptor and validate it against requested schema type
Args:
descriptor - yaml descriptor or path to a descriptor to be loaded
Returns descriptor as a dictionary
"""
if not os.path.exists(descriptor):
logger.debug("Descriptor path '%s' doesn't exists, trying to parse it directly."
% descriptor)
try:
return yaml.safe_load(descriptor)
except Exception as ex:
raise ConcreateError('Cannot load descriptor.', ex)
logger.debug("Loading descriptor from path '%s'." % descriptor)
with open(descriptor, 'r') as fh:
return yaml.safe_load(fh)
def __check_sum(self, target, algorithm, expected):
""" Check that file chksum is correct
Args:
alg - algorithm which will be used for diget
expected_chksum - checksum which artifact must mathc
"""
logger.debug("Checking '%s' %s hash..." % (self.name, algorithm))
hash_function = getattr(hashlib, algorithm)()
with open(target, "rb") as f:
for chunk in iter(lambda: f.read(65536), b""):
hash_function.update(chunk)
checksum = hash_function.hexdigest()
if checksum.lower() != expected.lower():
raise ConcreateError("The %s computed for the '%s' file ('%s') doesn't match the '%s' value" # noqa: E501
% (algorithm, self.name, checksum, expected))
logger.debug("Hash is correct.")
def prepare_repositories(self):
"""Udates descriptor with added repositories"""
configured_repositories = tools.cfg.get('repositories', {})
# We need to remove the custom "__name__" element before we can show
# which repository keys are defined in the configuration
configured_repository_names = configured_repositories.keys()
if '__name__' in configured_repository_names:
configured_repository_names.remove('__name__')
added_repos = []
target_dir = os.path.join(self.target, 'image', 'repos')
for repo in self.descriptor.get('packages',
{}).get('repositories', []):
if repo not in configured_repositories:
raise ConcreateError(
"Package repository '%s' used in descriptor is not "
"available in Concreate configuration file. "
"Available repositories: %s" %
(repo, configured_repository_names))
urls = configured_repositories[repo]
if urls:
# we need to do this in this cycle to prevent creation of empty dir
if not os.path.exists(target_dir):
os.makedirs(target_dir)
logger.info("Handling additional repository files...")
for url in urls.split(','):
Resource.new({'url': url}).copy(target_dir)
added_repos.append(
os.path.splitext(os.path.basename(url))[0])
logger.debug("Additional repository files handled")
self.descriptor['additional_repos'] = added_repos
def build(self, build_args):
raise ConcreateError("Buider.build() is not implemented!")
