def _update_src(self, build_config): """Retrieve and/or check the existence of the files needed for the build. This can include pulling from URL's. :param dict build_config: The build configuration dictionary. :returns: src_path, extra_files """ src_loc = build_config.get('source_location') if src_loc is None: return None # For URL's, check if the file needs to be updated, and try to do so. if self._isurl(src_loc): missing_libs = wget.missing_libs() if missing_libs: raise TestRunError( "The dependencies needed for remote source retrieval " "({}) are not available on this system. Please provide " "your test source locally.".format( ', '.join(missing_libs))) dwn_name = build_config.get('source_download_name') src_dest = self._download_path(src_loc, dwn_name) wget.update(self._pav_cfg, src_loc, src_dest) return src_dest src_path = self._find_file(Path(src_loc), 'test_src') if src_path is None: raise TestRunError( "Could not find and update src location '{}'".format(src_loc)) if src_path.is_dir(): # For directories, update the directories mtime to match the # latest mtime in the entire directory. self._date_dir(src_path) return src_path elif src_path.is_file(): # For static files, we'll end up just hashing the whole thing. return src_path else: raise TestRunError("Source location '{}' points to something " "unusable.".format(src_path))
def test_update(self): dest_fn = Path(tempfile.mktemp(dir='/tmp')) info_fn = dest_fn.with_suffix(dest_fn.suffix + '.info') self.assertFalse(dest_fn.exists()) self.assertFalse(info_fn.exists()) # Update should get the file if it doesn't exist. wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) self.assertTrue(dest_fn.exists()) self.assertTrue(info_fn.exists()) # It should update the file if the info file isn't there and the # sizes don't match. ctime = dest_fn.stat().st_ctime with dest_fn.open('ab') as dest_file: dest_file.write(b'a') info_fn.unlink() wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) ctime = new_ctime # We'll muck up the info file data, to force an update. db_data = {'ETag': 'nope', 'Content-Length': '-1'} with info_fn.open('w') as info_file: json.dump(db_data, info_file) wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) dest_fn.stat() info_fn.stat()
def _update_src(self, build_config): """Retrieve and/or check the existence of the files needed for the build. This can include pulling from URL's. :param dict build_config: The build configuration dictionary. :returns: src_path, extra_files """ src_loc = build_config.get('source_location') if src_loc is None: return None # For URL's, check if the file needs to be updated, and try to do so. if self._isurl(src_loc): dwn_name = build_config.get('source_download_name') src_dest = self._download_path(src_loc, dwn_name) wget.update(self._pav_cfg, src_loc, src_dest) return src_dest src_path = self._find_file(src_loc, 'test_src') if src_path is None: raise PavTestError("Could not find and update src location '{}'" .format(src_loc)) if os.path.isdir(src_path): # For directories, update the directories mtime to match the # latest mtime in the entire directory. self._date_dir(src_path) return src_path elif os.path.isfile(src_path): # For static files, we'll end up just hashing the whole thing. return src_path else: raise PavTestError("Source location '{}' points to something " "unusable.".format(src_path))
def test_update(self): dest_fn = Path(tempfile.mktemp(dir='/tmp')) info_fn = wget._get_info_fn(dest_fn) self.assertFalse(dest_fn.exists()) self.assertFalse(info_fn.exists()) # Update should get the file if it doesn't exist. try: wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) except wget.WGetError as err: self.fail("Failed with: {}".format(err.args[0])) self.assertTrue(dest_fn.exists()) self.assertTrue(info_fn.exists()) # It should update the file if the info file isn't there and the # sizes don't match. ctime = dest_fn.stat().st_ctime with dest_fn.open('ab') as dest_file: dest_file.write(b'a') info_fn.unlink() try: wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) except wget.WGetError as err: self.fail("Failed with: {}".format(err.args[0])) new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) ctime = new_ctime # We'll muck up the info file data, to force an update. db_data = {'ETag': 'nope', 'Content-Length': '-1'} with info_fn.open('w') as info_file: json.dump(db_data, info_file) try: wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) except wget.WGetError as err: self.fail("Failed with: {}".format(err.args[0])) new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) dest_fn.stat() info_fn.stat()
def test_update(self): # Try to get a configuration from the testing pavilion.yaml file. try: pav_cfg = config.PavilionConfigLoader().load(open(self.PAV_CONFIG_PATH)) except FileNotFoundError: self._logger.error("Could not find pavilion config at '{}'. You'll probably need to " "setup the proxy information for this test." .format(self.PAV_CONFIG_PATH)) pav_cfg = config.PavilionConfigLoader().load_empty() dest_fn = tempfile.mktemp(dir='/tmp') info_fn = '{}.info'.format(dest_fn) self.assertFalse(os.path.exists(dest_fn)) self.assertFalse(os.path.exists(info_fn)) # Update should get the file if it doesn't exist. wget.update(pav_cfg, self.GET_TARGET, dest_fn) self.assertTrue(os.path.exists(dest_fn)) self.assertTrue(os.path.exists(info_fn)) # It should update the file if the info file isn't there and the sizes don't match. ctime = os.stat(dest_fn).st_ctime with open(dest_fn, 'ab') as dest_file: dest_file.write(b'a') os.unlink(info_fn) wget.update(pav_cfg, self.GET_TARGET, dest_fn) new_ctime = os.stat(dest_fn).st_ctime self.assertNotEqual(new_ctime, ctime) ctime = new_ctime # We'll muck up the info file data, to force an update. with dbm.open(info_fn, 'w') as db: db['ETag'] = 'nope' db['Content-Length'] = '-1' wget.update(pav_cfg, self.GET_TARGET, dest_fn) new_ctime = os.stat(dest_fn).st_ctime self.assertNotEqual(new_ctime, ctime) os.unlink(dest_fn) os.unlink(info_fn)
def _update_src(self): """Retrieve and/or check the existence of the files needed for the build. This can include pulling from URL's. :returns: src_path, extra_files """ src_path = self._config.get('source_path') if src_path is None: # There is no source to do anything with. return None try: src_path = Path(src_path) except ValueError as err: raise TestBuilderError( "The source path must be a valid unix path, either relative " "or absolute, got '{}':\n{}".format(src_path, err.args[0])) found_src_path = self._find_file(src_path, 'test_src') src_url = self._config.get('source_url') src_download = self._config.get('source_download') if (src_url is not None and ((src_download == 'missing' and found_src_path is None) or src_download == 'latest')): # Make sure we have the library support to perform a download. missing_libs = wget.missing_libs() if missing_libs: raise TestBuilderError( "The dependencies needed for remote source retrieval " "({}) are not available on this system. Please provide " "your test source locally.".format( ', '.join(missing_libs))) if not src_path.is_absolute(): dwn_dest = self.test.suite_path.parents[ 1] / 'test_src' / src_path else: dwn_dest = src_path if not src_path.parent.exists(): try: src_path.parent.mkdir(parents=True) except OSError as err: raise TestBuilderError( "Could not create parent directory to place " "downloaded source:\n{}".format(err.args[0])) self.tracker.update( "Updating source at '{}'.".format(found_src_path), STATES.BUILDING) try: wget.update(self._pav_cfg, src_url, dwn_dest) except wget.WGetError as err: raise TestBuilderError( "Could not retrieve source from the given url '{}':\n{}". format(src_url, err.args[0])) return dwn_dest if found_src_path is None: raise TestBuilderError("Could not find source '{}'".format( src_path.as_posix())) if found_src_path.is_dir(): # For directories, update the directories mtime to match the # latest mtime in the entire directory. self._date_dir(found_src_path) return found_src_path elif found_src_path.is_file(): # For static files, we'll end up just hashing the whole thing. return found_src_path else: raise TestBuilderError( "Source location '{}' points to something unusable.".format( found_src_path))