def checkDigest(self, downloadRetriesLeft=3): CraftCore.log.debug("ArchiveSource.checkDigest called") filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): CraftCore.log.debug("check digests urls") if not CraftHash.checkFilesDigests(self.__downloadDir, filenames): CraftCore.log.error("invalid digest file") redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice( "Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes") if redownload: for filename in filenames: CraftCore.log.info( f"Deleting downloaded file: {filename}") utils.deleteFile( os.path.join(self.__downloadDir, filename)) for digestAlgorithm, digestFileEnding in CraftHash.HashAlgorithm.fileEndings( ).items(): digestFileName = filename + digestFileEnding if os.path.exists( os.path.join(self.__downloadDir, digestFileName)): CraftCore.log.info( f"Deleting downloaded file: {digestFileName}" ) utils.deleteFile( os.path.join(self.__downloadDir, digestFileName)) return self.fetch() and self.checkDigest( downloadRetriesLeft - 1) return False elif self.subinfo.hasTargetDigests(): CraftCore.log.debug("check digests") digests, algorithm = self.subinfo.targetDigest() if not CraftHash.checkFilesDigests(self.__downloadDir, filenames, digests, algorithm): CraftCore.log.error("invalid digest file") redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice( "Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes") if redownload: for filename in filenames: CraftCore.log.info( f"Deleting downloaded file: {filename}") utils.deleteFile( os.path.join(self.__downloadDir, filename)) return self.fetch() and self.checkDigest( downloadRetriesLeft - 1) return False else: CraftCore.log.debug("print source file digests") CraftHash.printFilesDigests( self.__downloadDir, filenames, self.subinfo.buildTarget, algorithm=CraftHash.HashAlgorithm.SHA256) return True
def checkDigest(self): craftDebug.log.debug("ArchiveSource.checkDigest called") filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): craftDebug.log.debug("check digests urls") if not CraftHash.checkFilesDigests(CraftStandardDirs.downloadDir(), filenames): craftDebug.log.error("invalid digest file") return False elif self.subinfo.hasTargetDigests(): craftDebug.log.debug("check digests") digests, algorithm = self.subinfo.targetDigest() if not CraftHash.checkFilesDigests(CraftStandardDirs.downloadDir(), filenames, digests, algorithm): craftDebug.log.error("invalid digest file") return False else: craftDebug.log.debug("print source file digests") CraftHash.printFilesDigests( CraftStandardDirs.downloadDir(), filenames, self.subinfo.buildTarget, algorithm=CraftHash.HashAlgorithm.SHA256) return True
def checkDigest(self, downloadRetriesLeft=3): CraftCore.log.debug("ArchiveSource.checkDigest called") filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): CraftCore.log.debug("check digests urls") if not CraftHash.checkFilesDigests(self.__downloadDir, filenames): CraftCore.log.error("invalid digest file") return self.__retry(downloadRetriesLeft, self.__redownload, filenames=filenames) elif self.subinfo.hasTargetDigests(): CraftCore.log.debug("check digests") digests, algorithm = self.subinfo.targetDigest() if not CraftHash.checkFilesDigests(self.__downloadDir, filenames, digests, algorithm): CraftCore.log.error("invalid digest file") return self.__retry(downloadRetriesLeft, self.__redownload, filenames=filenames) else: CraftCore.log.debug("print source file digests") CraftHash.printFilesDigests( self.__downloadDir, filenames, self.subinfo.buildTarget, algorithm=CraftHash.HashAlgorithm.SHA256) return True
def fetchBinary(self) -> bool: if self.subinfo.options.package.disableBinaryCache: return False archiveName = self.binaryArchiveName(includePackagePath=True) archvieFolder, localArchiveName = self.binaryArchiveName(includePackagePath=True).rsplit("/", 1) downloadFolder = os.path.join(self.cacheLocation(), archvieFolder) print(downloadFolder) if not os.path.exists(downloadFolder): os.makedirs(downloadFolder) for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): craftDebug.log.debug(f"Trying to restore {archiveName} from cache: {url}.") cache = utils.utilsCache.cacheJsonFromUrl(f"{url}/manifest.json") if not cache or not str(self) in cache or not archiveName in cache[str(self)]: continue if url != self.cacheLocation(): if not os.path.exists(os.path.join(downloadFolder, localArchiveName)): if not utils.getFile(f"{url}/{archiveName}", downloadFolder, localArchiveName): return False return CraftHash.checkFilesDigests(downloadFolder, [localArchiveName], digests=cache[str(self)][archiveName]["checksum"], digestAlgorithm=CraftHash.HashAlgorithm.SHA256) and \ self.cleanImage() \ and utils.unpackFile(downloadFolder, localArchiveName, self.imageDir()) \ and self.qmerge() return False
def fetchBinary(self) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) latest = None if not files: CraftCore.log.debug(f"Could not find {self}={self.version} in {url}") continue latest = files[0] if url != self.cacheLocation(): downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath(os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split(localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fUrl = f"{url}/{latest.fileName}" if not utils.getFile(fUrl, localArchivePath, localArchiveName): CraftCore.log.warning(f"Failed to fetch {fUrl}") return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests(localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): CraftCore.log.warning(f"Hash did not match, {localArchiveName} might be corrupted") return False self.subinfo.buildPrefix = latest.buildPrefix if not (self.cleanImage() and utils.unpackFile(localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False
def __fetchFromArchiveCache(self, downloadRetriesLeft : int=3): for url, files in self._getFileInfoFromArchiveCache(): self.__downloadDir.mkdir(parents=True, exist_ok=True) for entry in files: if entry.version != self.buildTarget: continue if not GetFiles.getFile(utils.urljoin(url, entry.fileName), self.__archiveDir, entry.fileName): self.__retry(downloadRetriesLeft, self.__fetchFromArchiveCache) if not CraftHash.checkFilesDigests(self.__archiveDir, [entry.fileName], digests=entry.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): return self.__retry(downloadRetriesLeft, lambda downloadRetriesLeft: utils.deleteFile(self.__archiveDir / entry.fileName) and self.__fetchFromArchiveCache(downloadRetriesLeft)) if self.__checkFilesPresent(self.localFileNames()): return True return False
def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson( CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) if not files: CraftCore.log.info( f"Could not find {self}={self.version} in {url}") continue latest = files[0] if not self.subinfo.options.dynamic.compatible( latest.config, latest.configHash): CraftCore.log.info( "Failed to restore package, configuration missmatch") CraftCore.debug.debug_line() CraftCore.log.info("Cached config: {}".format(", ".join( f"{k}={v}" for k, v in latest.config.items()))) CraftCore.log.info( f"Local config: {self.subinfo.options.dynamic}") CraftCore.debug.debug_line() # try next cache continue # if we are creating the cache, a rebuild on a failed fetch would be suboptimal createingCache = CraftCore.settings.getboolean( "Packager", "CreateCache", False) if url != self.cacheLocation(): downloadFolder = self.cacheLocation( os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath( os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split( localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fileName = latest.fileName if CraftCore.compiler.isWindows: fileName = fileName.replace("\\", "/") fUrl = f"{url}/{fileName}" # try it up to 3 times retries = 3 while True: if GetFiles.getFile(fUrl, localArchivePath, localArchiveName): break msg = f"Failed to fetch {fUrl}" retries -= 1 if not retries: if createingCache: raise BlueprintException(msg, self.package) else: CraftCore.log.warning(msg) return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests( localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): msg = f"Hash did not match, {localArchiveName} might be corrupted" CraftCore.log.warning(msg) if downloadRetriesLeft and CraftChoicePrompt.promptForChoice( "Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile( localArchiveAbsPath) and self.fetchBinary( downloadRetriesLeft=downloadRetriesLeft - 1) if createingCache: raise BlueprintException(msg, self.package) return False self.subinfo.buildPrefix = latest.buildPrefix self.subinfo.isCachedBuild = True if not (self.cleanImage() and utils.unpackFile( localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False
def hashTest(self, hash, algorithm): path, name = os.path.split(self.tmpFile) self.assertEquals(CraftHash.checkFilesDigests(path, [name], hash, algorithm), True)