def __init__(self, pav_cfg, test, mb_tracker, build_name=None):
"""Inititalize the build object.
:param pav_cfg: The Pavilion config object
:param pavilion.test_run.TestRun test: The test run responsible for
starting this build.
:param MultiBuildTracker mb_tracker: A thread-safe tracker object for
keeping info on what the build is doing.
:param str build_name: The build name, if this is a build that already
exists.
:raises TestBuilderError: When the builder can't be initialized.
"""
if mb_tracker is None:
mb_tracker = MultiBuildTracker(log=False)
self.tracker = mb_tracker.register(self, test.status)
self._pav_cfg = pav_cfg
self._config = test.config.get('build', {})
self._group = test.group
self._umask = test.umask
self._script_path = test.build_script_path
self.test = test
self._timeout = test.build_timeout
self._timeout_file = test.build_timeout_file
self._fix_source_path()
if not test.build_local:
self.tracker.update(state=STATES.BUILD_DEFERRED,
note="Build will run on nodes.")
if build_name is None:
self.name = self.name_build()
self.tracker.update(state=STATES.BUILD_CREATED,
note="Builder created.")
else:
self.name = build_name
self.path = pav_cfg.working_dir / 'builds' / self.name # type: Path
self.tmp_log_path = self.path.with_suffix('.log')
self.log_path = self.path / self.LOG_NAME
fail_name = 'fail.{}.{}'.format(self.name, self.test.id)
self.fail_path = pav_cfg.working_dir / 'builds' / fail_name
self.finished_path = self.path.with_suffix(self.FINISHED_SUFFIX)
if self._timeout_file is not None:
self._timeout_file = self.path / self._timeout_file
else:
self._timeout_file = self.tmp_log_path
# Don't allow a file to be written outside of the build context dir.
files_to_create = self._config.get('create_files')
if files_to_create:
for file, contents in files_to_create.items():
file_path = Path(utils.resolve_path(self.path / file))
if not utils.dir_contains(file_path,
utils.resolve_path(self.path)):
raise TestBuilderError(
"'create_file: {}': file path"
" outside build context.".format(file_path))
def filter_series(path: Path) -> bool:
"""Return True if the series does not have any valid symlinked tests.
"""
for test_path in path.iterdir():
if (test_path.is_symlink() and test_path.exists()
and utils.resolve_path(test_path).exists()):
return False
return True
def _get_used_build_paths(tests_dir: Path) -> set:
"""Generate a set of all build paths currently used by one or more test
runs."""
used_builds = set()
for path in dir_db.select(tests_dir).paths:
build_origin_symlink = path / 'build_origin'
build_origin = None
if (build_origin_symlink.exists()
and build_origin_symlink.is_symlink()
and utils.resolve_path(build_origin_symlink).exists()):
build_origin = build_origin_symlink.resolve()
if build_origin is not None:
used_builds.add(build_origin.name)
return used_builds
def _setup_build_dir(self, dest):
"""Setup the build directory, by extracting or copying the source
and any extra files.
:param dest: Path to the intended build directory. This is generally a
temporary location.
:return: None
"""
raw_src_path = self._config.get('source_path')
if raw_src_path is None:
src_path = None
else:
src_path = self._find_file(Path(raw_src_path), 'test_src')
if src_path is None:
raise TestBuilderError(
"Could not find source file '{}'".format(raw_src_path))
# Resolve any softlinks to get the real file.
src_path = src_path.resolve()
if src_path is None:
# If there is no source archive or data, just make the build
# directory.
dest.mkdir()
elif src_path.is_dir():
# Recursively copy the src directory to the build directory.
self.tracker.update(
state=STATES.BUILDING,
note=("Copying source directory {} for build {} "
"as the build directory.".format(src_path, dest)))
shutil.copytree(src_path.as_posix(),
dest.as_posix(),
symlinks=True)
elif src_path.is_file():
# Handle decompression of a stream compressed file. The interfaces
# for the libs are all the same; we just have to choose the right
# one to use. Zips are handled as an archive, below.
category, subtype = utils.get_mime_type(src_path)
if category == 'application' and subtype in self.TAR_SUBTYPES:
if tarfile.is_tarfile(src_path.as_posix()):
self.tracker.update(
state=STATES.BUILDING,
note=("Extracting tarfile {} for build {}".format(
src_path, dest)))
extract.extract_tarball(src_path, dest)
else:
self.tracker.update(
state=STATES.BUILDING,
note=("Extracting {} file {} for build {} into the "
"build directory.".format(
subtype, src_path, dest)))
extract.decompress_file(src_path, dest, subtype)
elif category == 'application' and subtype == 'zip':
self.tracker.update(
state=STATES.BUILDING,
note=("Extracting zip file {} for build {}.".format(
src_path, dest)))
extract.unzip_file(src_path, dest)
else:
# Finally, simply copy any other types of files into the build
# directory.
self.tracker.update(
state=STATES.BUILDING,
note="Copying file {} for build {} into the build "
"directory.".format(src_path, dest))
copy_dest = dest / src_path.name
try:
dest.mkdir()
shutil.copy(src_path.as_posix(), copy_dest.as_posix())
except OSError as err:
raise TestBuilderError(
"Could not copy test src '{}' to '{}': {}".format(
src_path, dest, err))
# Create build time file(s).
files_to_create = self._config.get('create_files')
if files_to_create:
for file, contents in files_to_create.items():
file_path = Path(utils.resolve_path(dest / file))
# Do not allow file to clash with existing directory.
if file_path.is_dir():
raise TestBuilderError(
"'create_file: {}' clashes with"
" existing directory in test source.".format(
str(file_path)))
dirname = file_path.parent
(dest / dirname).mkdir(parents=True, exist_ok=True)
with file_path.open('w') as file_:
for line in contents:
file_.write("{}\n".format(line))
# Now we just need to copy over all of the extra files.
for extra in self._config.get('extra_files', []):
extra = Path(extra)
path = self._find_file(extra, 'test_src')
final_dest = dest / path.name
try:
shutil.copy(path.as_posix(), final_dest.as_posix())
except OSError as err:
raise TestBuilderError(
"Could not copy extra file '{}' to dest '{}': {}".format(
path, dest, err))