def checkDigest(self, downloadRetriesLeft=3):
CraftCore.log.debug("ArchiveSource.checkDigest called")
filenames = self.localFileNames()
if self.subinfo.hasTargetDigestUrls():
CraftCore.log.debug("check digests urls")
if not CraftHash.checkFilesDigests(self.__downloadDir, filenames):
CraftCore.log.error("invalid digest file")
redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice(
"Do you want to delete the files and redownload them?",
[("Yes", True), ("No", False)],
default="Yes")
if redownload:
for filename in filenames:
CraftCore.log.info(
f"Deleting downloaded file: {filename}")
utils.deleteFile(
os.path.join(self.__downloadDir, filename))
for digestAlgorithm, digestFileEnding in CraftHash.HashAlgorithm.fileEndings(
).items():
digestFileName = filename + digestFileEnding
if os.path.exists(
os.path.join(self.__downloadDir,
digestFileName)):
CraftCore.log.info(
f"Deleting downloaded file: {digestFileName}"
)
utils.deleteFile(
os.path.join(self.__downloadDir,
digestFileName))
return self.fetch() and self.checkDigest(
downloadRetriesLeft - 1)
return False
elif self.subinfo.hasTargetDigests():
CraftCore.log.debug("check digests")
digests, algorithm = self.subinfo.targetDigest()
if not CraftHash.checkFilesDigests(self.__downloadDir, filenames,
digests, algorithm):
CraftCore.log.error("invalid digest file")
redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice(
"Do you want to delete the files and redownload them?",
[("Yes", True), ("No", False)],
default="Yes")
if redownload:
for filename in filenames:
CraftCore.log.info(
f"Deleting downloaded file: {filename}")
utils.deleteFile(
os.path.join(self.__downloadDir, filename))
return self.fetch() and self.checkDigest(
downloadRetriesLeft - 1)
return False
else:
CraftCore.log.debug("print source file digests")
CraftHash.printFilesDigests(
self.__downloadDir,
filenames,
self.subinfo.buildTarget,
algorithm=CraftHash.HashAlgorithm.SHA256)
return True
def checkDigest(self):
craftDebug.log.debug("ArchiveSource.checkDigest called")
filenames = self.localFileNames()
if self.subinfo.hasTargetDigestUrls():
craftDebug.log.debug("check digests urls")
if not CraftHash.checkFilesDigests(CraftStandardDirs.downloadDir(),
filenames):
craftDebug.log.error("invalid digest file")
return False
elif self.subinfo.hasTargetDigests():
craftDebug.log.debug("check digests")
digests, algorithm = self.subinfo.targetDigest()
if not CraftHash.checkFilesDigests(CraftStandardDirs.downloadDir(),
filenames, digests, algorithm):
craftDebug.log.error("invalid digest file")
return False
else:
craftDebug.log.debug("print source file digests")
CraftHash.printFilesDigests(
CraftStandardDirs.downloadDir(),
filenames,
self.subinfo.buildTarget,
algorithm=CraftHash.HashAlgorithm.SHA256)
return True
def checkDigest(self, downloadRetriesLeft=3):
CraftCore.log.debug("ArchiveSource.checkDigest called")
filenames = self.localFileNames()
if self.subinfo.hasTargetDigestUrls():
CraftCore.log.debug("check digests urls")
if not CraftHash.checkFilesDigests(self.__downloadDir, filenames):
CraftCore.log.error("invalid digest file")
return self.__retry(downloadRetriesLeft,
self.__redownload,
filenames=filenames)
elif self.subinfo.hasTargetDigests():
CraftCore.log.debug("check digests")
digests, algorithm = self.subinfo.targetDigest()
if not CraftHash.checkFilesDigests(self.__downloadDir, filenames,
digests, algorithm):
CraftCore.log.error("invalid digest file")
return self.__retry(downloadRetriesLeft,
self.__redownload,
filenames=filenames)
else:
CraftCore.log.debug("print source file digests")
CraftHash.printFilesDigests(
self.__downloadDir,
filenames,
self.subinfo.buildTarget,
algorithm=CraftHash.HashAlgorithm.SHA256)
return True
def fetchBinary(self) -> bool:
if self.subinfo.options.package.disableBinaryCache:
return False
archiveName = self.binaryArchiveName(includePackagePath=True)
archvieFolder, localArchiveName = self.binaryArchiveName(includePackagePath=True).rsplit("/", 1)
downloadFolder = os.path.join(self.cacheLocation(), archvieFolder)
print(downloadFolder)
if not os.path.exists(downloadFolder):
os.makedirs(downloadFolder)
for url in [self.cacheLocation()] + self.cacheRepositoryUrls():
craftDebug.log.debug(f"Trying to restore {archiveName} from cache: {url}.")
cache = utils.utilsCache.cacheJsonFromUrl(f"{url}/manifest.json")
if not cache or not str(self) in cache or not archiveName in cache[str(self)]:
continue
if url != self.cacheLocation():
if not os.path.exists(os.path.join(downloadFolder, localArchiveName)):
if not utils.getFile(f"{url}/{archiveName}", downloadFolder, localArchiveName):
return False
return CraftHash.checkFilesDigests(downloadFolder, [localArchiveName],
digests=cache[str(self)][archiveName]["checksum"],
digestAlgorithm=CraftHash.HashAlgorithm.SHA256) and \
self.cleanImage() \
and utils.unpackFile(downloadFolder, localArchiveName, self.imageDir()) \
and self.qmerge()
return False
def fetchBinary(self) -> bool:
if self.subinfo.options.package.disableBinaryCache:
return False
for url in [self.cacheLocation()] + self.cacheRepositoryUrls():
CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.")
if url == self.cacheLocation():
fileUrl = f"{url}/manifest.json"
if os.path.exists(fileUrl):
with open(fileUrl, "rt", encoding="UTF-8") as f:
manifest = CraftManifest.fromJson(json.load(f))
else:
continue
else:
manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json"))
fileEntry = manifest.get(str(self)).files
files = []
for f in fileEntry:
if f.version == self.version:
files.append(f)
latest = None
if not files:
CraftCore.log.debug(f"Could not find {self}={self.version} in {url}")
continue
latest = files[0]
if url != self.cacheLocation():
downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache"))
else:
downloadFolder = self.cacheLocation()
localArchiveAbsPath = OsUtils.toNativePath(os.path.join(downloadFolder, latest.fileName))
localArchivePath, localArchiveName = os.path.split(localArchiveAbsPath)
if url != self.cacheLocation():
if not os.path.exists(localArchiveAbsPath):
os.makedirs(localArchivePath, exist_ok=True)
fUrl = f"{url}/{latest.fileName}"
if not utils.getFile(fUrl, localArchivePath, localArchiveName):
CraftCore.log.warning(f"Failed to fetch {fUrl}")
return False
elif not os.path.isfile(localArchiveAbsPath):
continue
if not CraftHash.checkFilesDigests(localArchivePath, [localArchiveName],
digests=latest.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
CraftCore.log.warning(f"Hash did not match, {localArchiveName} might be corrupted")
return False
self.subinfo.buildPrefix = latest.buildPrefix
if not (self.cleanImage()
and utils.unpackFile(localArchivePath, localArchiveName, self.imageDir())
and self.internalPostInstall()
and self.postInstall()
and self.qmerge()
and self.internalPostQmerge()
and self.postQmerge()):
return False
return True
return False
def __fetchFromArchiveCache(self, downloadRetriesLeft : int=3):
for url, files in self._getFileInfoFromArchiveCache():
self.__downloadDir.mkdir(parents=True, exist_ok=True)
for entry in files:
if entry.version != self.buildTarget:
continue
if not GetFiles.getFile(utils.urljoin(url, entry.fileName), self.__archiveDir, entry.fileName):
self.__retry(downloadRetriesLeft, self.__fetchFromArchiveCache)
if not CraftHash.checkFilesDigests(self.__archiveDir, [entry.fileName],
digests=entry.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
return self.__retry(downloadRetriesLeft,
lambda downloadRetriesLeft: utils.deleteFile(self.__archiveDir / entry.fileName) and self.__fetchFromArchiveCache(downloadRetriesLeft))
if self.__checkFilesPresent(self.localFileNames()):
return True
return False
def fetchBinary(self, downloadRetriesLeft=3) -> bool:
if self.subinfo.options.package.disableBinaryCache:
return False
for url in [self.cacheLocation()] + self.cacheRepositoryUrls():
CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.")
if url == self.cacheLocation():
fileUrl = f"{url}/manifest.json"
if os.path.exists(fileUrl):
with open(fileUrl, "rt", encoding="UTF-8") as f:
manifest = CraftManifest.fromJson(json.load(f))
else:
continue
else:
manifest = CraftManifest.fromJson(
CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json"))
fileEntry = manifest.get(str(self)).files
files = []
for f in fileEntry:
if f.version == self.version:
files.append(f)
if not files:
CraftCore.log.info(
f"Could not find {self}={self.version} in {url}")
continue
latest = files[0]
if not self.subinfo.options.dynamic.compatible(
latest.config, latest.configHash):
CraftCore.log.info(
"Failed to restore package, configuration missmatch")
CraftCore.debug.debug_line()
CraftCore.log.info("Cached config: {}".format(", ".join(
f"{k}={v}" for k, v in latest.config.items())))
CraftCore.log.info(
f"Local config: {self.subinfo.options.dynamic}")
CraftCore.debug.debug_line()
# try next cache
continue
# if we are creating the cache, a rebuild on a failed fetch would be suboptimal
createingCache = CraftCore.settings.getboolean(
"Packager", "CreateCache", False)
if url != self.cacheLocation():
downloadFolder = self.cacheLocation(
os.path.join(CraftCore.standardDirs.downloadDir(),
"cache"))
else:
downloadFolder = self.cacheLocation()
localArchiveAbsPath = OsUtils.toNativePath(
os.path.join(downloadFolder, latest.fileName))
localArchivePath, localArchiveName = os.path.split(
localArchiveAbsPath)
if url != self.cacheLocation():
if not os.path.exists(localArchiveAbsPath):
os.makedirs(localArchivePath, exist_ok=True)
fileName = latest.fileName
if CraftCore.compiler.isWindows:
fileName = fileName.replace("\\", "/")
fUrl = f"{url}/{fileName}"
# try it up to 3 times
retries = 3
while True:
if GetFiles.getFile(fUrl, localArchivePath,
localArchiveName):
break
msg = f"Failed to fetch {fUrl}"
retries -= 1
if not retries:
if createingCache:
raise BlueprintException(msg, self.package)
else:
CraftCore.log.warning(msg)
return False
elif not os.path.isfile(localArchiveAbsPath):
continue
if not CraftHash.checkFilesDigests(
localArchivePath, [localArchiveName],
digests=latest.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
msg = f"Hash did not match, {localArchiveName} might be corrupted"
CraftCore.log.warning(msg)
if downloadRetriesLeft and CraftChoicePrompt.promptForChoice(
"Do you want to delete the files and redownload them?",
[("Yes", True), ("No", False)],
default="Yes"):
return utils.deleteFile(
localArchiveAbsPath) and self.fetchBinary(
downloadRetriesLeft=downloadRetriesLeft - 1)
if createingCache:
raise BlueprintException(msg, self.package)
return False
self.subinfo.buildPrefix = latest.buildPrefix
self.subinfo.isCachedBuild = True
if not (self.cleanImage() and utils.unpackFile(
localArchivePath, localArchiveName, self.imageDir())
and self.internalPostInstall() and self.postInstall()
and self.qmerge() and self.internalPostQmerge()
and self.postQmerge()):
return False
return True
return False
def hashTest(self, hash, algorithm):
path, name = os.path.split(self.tmpFile)
self.assertEquals(CraftHash.checkFilesDigests(path, [name], hash, algorithm), True)