def _update_src(self, build_config):
"""Retrieve and/or check the existence of the files needed for the
build. This can include pulling from URL's.
:param dict build_config: The build configuration dictionary.
:returns: src_path, extra_files
"""
src_loc = build_config.get('source_location')
if src_loc is None:
return None
# For URL's, check if the file needs to be updated, and try to do so.
if self._isurl(src_loc):
missing_libs = wget.missing_libs()
if missing_libs:
raise TestRunError(
"The dependencies needed for remote source retrieval "
"({}) are not available on this system. Please provide "
"your test source locally.".format(
', '.join(missing_libs)))
dwn_name = build_config.get('source_download_name')
src_dest = self._download_path(src_loc, dwn_name)
wget.update(self._pav_cfg, src_loc, src_dest)
return src_dest
src_path = self._find_file(Path(src_loc), 'test_src')
if src_path is None:
raise TestRunError(
"Could not find and update src location '{}'".format(src_loc))
if src_path.is_dir():
# For directories, update the directories mtime to match the
# latest mtime in the entire directory.
self._date_dir(src_path)
return src_path
elif src_path.is_file():
# For static files, we'll end up just hashing the whole thing.
return src_path
else:
raise TestRunError("Source location '{}' points to something "
"unusable.".format(src_path))
def test_update(self):
dest_fn = Path(tempfile.mktemp(dir='/tmp'))
info_fn = dest_fn.with_suffix(dest_fn.suffix + '.info')
self.assertFalse(dest_fn.exists())
self.assertFalse(info_fn.exists())
# Update should get the file if it doesn't exist.
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
self.assertTrue(dest_fn.exists())
self.assertTrue(info_fn.exists())
# It should update the file if the info file isn't there and the
# sizes don't match.
ctime = dest_fn.stat().st_ctime
with dest_fn.open('ab') as dest_file:
dest_file.write(b'a')
info_fn.unlink()
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
new_ctime = dest_fn.stat().st_ctime
self.assertNotEqual(new_ctime, ctime)
ctime = new_ctime
# We'll muck up the info file data, to force an update.
db_data = {'ETag': 'nope', 'Content-Length': '-1'}
with info_fn.open('w') as info_file:
json.dump(db_data, info_file)
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
new_ctime = dest_fn.stat().st_ctime
self.assertNotEqual(new_ctime, ctime)
dest_fn.stat()
info_fn.stat()
def _update_src(self, build_config):
"""Retrieve and/or check the existence of the files needed for the
build. This can include pulling from URL's.
:param dict build_config: The build configuration dictionary.
:returns: src_path, extra_files
"""
src_loc = build_config.get('source_location')
if src_loc is None:
return None
# For URL's, check if the file needs to be updated, and try to do so.
if self._isurl(src_loc):
dwn_name = build_config.get('source_download_name')
src_dest = self._download_path(src_loc, dwn_name)
wget.update(self._pav_cfg, src_loc, src_dest)
return src_dest
src_path = self._find_file(src_loc, 'test_src')
if src_path is None:
raise PavTestError("Could not find and update src location '{}'"
.format(src_loc))
if os.path.isdir(src_path):
# For directories, update the directories mtime to match the
# latest mtime in the entire directory.
self._date_dir(src_path)
return src_path
elif os.path.isfile(src_path):
# For static files, we'll end up just hashing the whole thing.
return src_path
else:
raise PavTestError("Source location '{}' points to something "
"unusable.".format(src_path))
def test_update(self):
dest_fn = Path(tempfile.mktemp(dir='/tmp'))
info_fn = wget._get_info_fn(dest_fn)
self.assertFalse(dest_fn.exists())
self.assertFalse(info_fn.exists())
# Update should get the file if it doesn't exist.
try:
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
except wget.WGetError as err:
self.fail("Failed with: {}".format(err.args[0]))
self.assertTrue(dest_fn.exists())
self.assertTrue(info_fn.exists())
# It should update the file if the info file isn't there and the
# sizes don't match.
ctime = dest_fn.stat().st_ctime
with dest_fn.open('ab') as dest_file:
dest_file.write(b'a')
info_fn.unlink()
try:
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
except wget.WGetError as err:
self.fail("Failed with: {}".format(err.args[0]))
new_ctime = dest_fn.stat().st_ctime
self.assertNotEqual(new_ctime, ctime)
ctime = new_ctime
# We'll muck up the info file data, to force an update.
db_data = {'ETag': 'nope', 'Content-Length': '-1'}
with info_fn.open('w') as info_file:
json.dump(db_data, info_file)
try:
wget.update(self.pav_cfg, self.GET_TARGET, dest_fn)
except wget.WGetError as err:
self.fail("Failed with: {}".format(err.args[0]))
new_ctime = dest_fn.stat().st_ctime
self.assertNotEqual(new_ctime, ctime)
dest_fn.stat()
info_fn.stat()
def test_update(self):
# Try to get a configuration from the testing pavilion.yaml file.
try:
pav_cfg = config.PavilionConfigLoader().load(open(self.PAV_CONFIG_PATH))
except FileNotFoundError:
self._logger.error("Could not find pavilion config at '{}'. You'll probably need to "
"setup the proxy information for this test."
.format(self.PAV_CONFIG_PATH))
pav_cfg = config.PavilionConfigLoader().load_empty()
dest_fn = tempfile.mktemp(dir='/tmp')
info_fn = '{}.info'.format(dest_fn)
self.assertFalse(os.path.exists(dest_fn))
self.assertFalse(os.path.exists(info_fn))
# Update should get the file if it doesn't exist.
wget.update(pav_cfg, self.GET_TARGET, dest_fn)
self.assertTrue(os.path.exists(dest_fn))
self.assertTrue(os.path.exists(info_fn))
# It should update the file if the info file isn't there and the sizes don't match.
ctime = os.stat(dest_fn).st_ctime
with open(dest_fn, 'ab') as dest_file:
dest_file.write(b'a')
os.unlink(info_fn)
wget.update(pav_cfg, self.GET_TARGET, dest_fn)
new_ctime = os.stat(dest_fn).st_ctime
self.assertNotEqual(new_ctime, ctime)
ctime = new_ctime
# We'll muck up the info file data, to force an update.
with dbm.open(info_fn, 'w') as db:
db['ETag'] = 'nope'
db['Content-Length'] = '-1'
wget.update(pav_cfg, self.GET_TARGET, dest_fn)
new_ctime = os.stat(dest_fn).st_ctime
self.assertNotEqual(new_ctime, ctime)
os.unlink(dest_fn)
os.unlink(info_fn)
def _update_src(self):
"""Retrieve and/or check the existence of the files needed for the
build. This can include pulling from URL's.
:returns: src_path, extra_files
"""
src_path = self._config.get('source_path')
if src_path is None:
# There is no source to do anything with.
return None
try:
src_path = Path(src_path)
except ValueError as err:
raise TestBuilderError(
"The source path must be a valid unix path, either relative "
"or absolute, got '{}':\n{}".format(src_path, err.args[0]))
found_src_path = self._find_file(src_path, 'test_src')
src_url = self._config.get('source_url')
src_download = self._config.get('source_download')
if (src_url is not None
and ((src_download == 'missing' and found_src_path is None)
or src_download == 'latest')):
# Make sure we have the library support to perform a download.
missing_libs = wget.missing_libs()
if missing_libs:
raise TestBuilderError(
"The dependencies needed for remote source retrieval "
"({}) are not available on this system. Please provide "
"your test source locally.".format(
', '.join(missing_libs)))
if not src_path.is_absolute():
dwn_dest = self.test.suite_path.parents[
1] / 'test_src' / src_path
else:
dwn_dest = src_path
if not src_path.parent.exists():
try:
src_path.parent.mkdir(parents=True)
except OSError as err:
raise TestBuilderError(
"Could not create parent directory to place "
"downloaded source:\n{}".format(err.args[0]))
self.tracker.update(
"Updating source at '{}'.".format(found_src_path),
STATES.BUILDING)
try:
wget.update(self._pav_cfg, src_url, dwn_dest)
except wget.WGetError as err:
raise TestBuilderError(
"Could not retrieve source from the given url '{}':\n{}".
format(src_url, err.args[0]))
return dwn_dest
if found_src_path is None:
raise TestBuilderError("Could not find source '{}'".format(
src_path.as_posix()))
if found_src_path.is_dir():
# For directories, update the directories mtime to match the
# latest mtime in the entire directory.
self._date_dir(found_src_path)
return found_src_path
elif found_src_path.is_file():
# For static files, we'll end up just hashing the whole thing.
return found_src_path
else:
raise TestBuilderError(
"Source location '{}' points to something unusable.".format(
found_src_path))
